481 lines
19 KiB
Python
481 lines
19 KiB
Python
"""Transferring, verifying, and removing active sources (US06-02).
|
|
|
|
Every case here asks the same question the service exists to answer: could an
|
|
original leave active storage without a durable, byte-identical archive copy? The
|
|
files are real, the hashes are real, and each failure path asserts that the source
|
|
is still exactly where it was.
|
|
|
|
Cross-filesystem behaviour is forced by patching the device comparison rather than
|
|
by requiring a second real filesystem in CI — the copy/verify/publish code that runs
|
|
is the production one.
|
|
"""
|
|
|
|
import json
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
from sqlalchemy import select
|
|
|
|
from photo_pipeline.api.app import create_app
|
|
from photo_pipeline.config import Config
|
|
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
|
|
from photo_pipeline.models import Asset, AssetPath, UploadBatch, UploadItem
|
|
from photo_pipeline.services import archive_transfer as transfer_module
|
|
from photo_pipeline.services.archive_journal import ArchiveState
|
|
from photo_pipeline.services.archive_transfer import (
|
|
MANIFEST_NAME,
|
|
ArchiveTransferService,
|
|
read_manifest,
|
|
)
|
|
from photo_pipeline.services.archives import ArchiveError, ArchiveService
|
|
from photo_pipeline.services.hashing import sha256_file
|
|
|
|
pytestmark = pytest.mark.phase_f # part of the Phase F acceptance gate (US06-06)
|
|
|
|
NOW = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
|
|
|
|
|
# ── environment ──────────────────────────────────────────────────────────────
|
|
|
|
|
|
def _env(tmp_path):
|
|
(tmp_path / "data").mkdir(exist_ok=True)
|
|
lib = tmp_path / "lib"
|
|
lib.mkdir(exist_ok=True)
|
|
archive = tmp_path / "archive"
|
|
archive.mkdir(exist_ok=True)
|
|
config = Config.from_env(
|
|
{
|
|
"PHOTO_PIPELINE_DATA_DIR": str(tmp_path / "data"),
|
|
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(lib),
|
|
"PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES": "0",
|
|
}
|
|
)
|
|
run_migrations(config.database_url)
|
|
return config, create_session_factory(create_db_engine(config.database_url)), lib, archive
|
|
|
|
|
|
def _album(sf, lib, album="rome", names=("a.jpg", "b.jpg")):
|
|
"""A real album whose assets carry the verified upload evidence archiving needs."""
|
|
folder = lib / album
|
|
folder.mkdir(parents=True, exist_ok=True)
|
|
ids = []
|
|
with sf() as session:
|
|
batch_id = str(uuid.uuid4())
|
|
session.add(
|
|
UploadBatch(
|
|
id=batch_id,
|
|
album=album,
|
|
folder=str(folder),
|
|
album_name=album,
|
|
state="succeeded",
|
|
preflight_token="v1:test",
|
|
outcome_state="verified",
|
|
created_at=NOW,
|
|
)
|
|
)
|
|
for name in names:
|
|
path = folder / name
|
|
path.write_bytes(f"{album}/{name} bytes".encode() * 8)
|
|
asset_id = str(uuid.uuid4())
|
|
ids.append(asset_id)
|
|
session.add(
|
|
Asset(
|
|
id=asset_id,
|
|
original_path=str(path),
|
|
current_path=str(path),
|
|
discovered_at=NOW,
|
|
hash_version=1,
|
|
byte_size=path.stat().st_size,
|
|
current_sha256=sha256_file(path),
|
|
)
|
|
)
|
|
session.add(AssetPath(asset_id=asset_id, path=str(path), valid_from=NOW))
|
|
session.add(
|
|
UploadItem(
|
|
batch_id=batch_id,
|
|
asset_id=asset_id,
|
|
path=str(path),
|
|
sha256=sha256_file(path),
|
|
sha1="0" * 40,
|
|
state="sent",
|
|
outcome="uploaded",
|
|
)
|
|
)
|
|
session.commit()
|
|
return folder, ids
|
|
|
|
|
|
def _location(sf, config, archive, name="external"):
|
|
return ArchiveService(sf, config=config).register(name, str(archive))
|
|
|
|
|
|
def _plan(sf, config, location_id, albums=None):
|
|
service = ArchiveTransferService(sf, config=config)
|
|
token = ArchiveService(sf, config=config).preflight(location_id, albums)["token"]
|
|
return service, service.create(location_id, albums, token=token)
|
|
|
|
|
|
def _contents(*roots):
|
|
"""Every file's bytes under the given roots, ignoring our own bookkeeping."""
|
|
return sorted(
|
|
path.read_bytes()
|
|
for root in roots
|
|
for path in root.rglob("*")
|
|
if path.is_file() and path.name != MANIFEST_NAME and not path.name.startswith(".")
|
|
)
|
|
|
|
|
|
def _states(service, plan_id):
|
|
return [row["journal_state"] for row in service.journal.operations(plan_id)]
|
|
|
|
|
|
def _asset(sf, asset_id):
|
|
with sf() as session:
|
|
return session.get(Asset, asset_id)
|
|
|
|
|
|
# ── planning ─────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_a_plan_records_every_file_with_its_expected_hash_and_moves_nothing(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
folder, ids = _album(sf, lib)
|
|
location = _location(sf, config, archive)
|
|
before = _contents(lib)
|
|
|
|
service, plan = _plan(sf, config, location["id"])
|
|
|
|
assert plan["state"] == "planned" and plan["asset_count"] == 2
|
|
assert sorted(op["asset_id"] for op in plan["operations"]) == sorted(ids)
|
|
for operation in plan["operations"]:
|
|
source = folder / operation["archive_path"].split("/")[-1]
|
|
assert operation["expected_sha256"] == sha256_file(source)
|
|
assert operation["archive_path"].startswith("rome/")
|
|
assert operation["journal_state"] == ArchiveState.PLANNED
|
|
assert _contents(lib) == before
|
|
assert _contents(archive) == []
|
|
|
|
|
|
def test_a_stale_token_cannot_create_a_plan(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
folder, _ = _album(sf, lib)
|
|
location = _location(sf, config, archive)
|
|
token = ArchiveService(sf, config=config).preflight(location["id"])["token"]
|
|
|
|
(folder / "a.jpg").write_bytes(b"edited after approval")
|
|
|
|
with pytest.raises(ArchiveError) as error:
|
|
ArchiveTransferService(sf, config=config).create(location["id"], token=token)
|
|
assert error.value.code == "stale_token"
|
|
|
|
|
|
def test_a_blocked_scope_cannot_create_a_plan(tmp_path):
|
|
"""No verified upload means Immich may not hold these bytes; archiving would
|
|
remove the only copy."""
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
folder = lib / "rome"
|
|
folder.mkdir()
|
|
path = folder / "a.jpg"
|
|
path.write_bytes(b"never uploaded")
|
|
with sf() as session:
|
|
session.add(
|
|
Asset(
|
|
id=str(uuid.uuid4()),
|
|
original_path=str(path),
|
|
current_path=str(path),
|
|
discovered_at=NOW,
|
|
hash_version=1,
|
|
byte_size=path.stat().st_size,
|
|
current_sha256=sha256_file(path),
|
|
)
|
|
)
|
|
session.commit()
|
|
location = _location(sf, config, archive)
|
|
token = ArchiveService(sf, config=config).preflight(location["id"])["token"]
|
|
|
|
with pytest.raises(ArchiveError) as error:
|
|
ArchiveTransferService(sf, config=config).create(location["id"], token=token)
|
|
|
|
assert error.value.code == "blocked"
|
|
assert path.exists()
|
|
|
|
|
|
# ── the transfer ─────────────────────────────────────────────────────────────
|
|
|
|
|
|
@pytest.mark.parametrize("same_filesystem", [True, False])
|
|
def test_archiving_verifies_the_copy_before_the_source_is_removed(
|
|
tmp_path, monkeypatch, same_filesystem
|
|
):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
folder, ids = _album(sf, lib)
|
|
location = _location(sf, config, archive)
|
|
service, plan = _plan(sf, config, location["id"])
|
|
expected = {op["asset_id"]: op["expected_sha256"] for op in plan["operations"]}
|
|
before = _contents(lib)
|
|
if not same_filesystem:
|
|
# Force the copy-verify-publish path without needing a second real volume.
|
|
monkeypatch.setattr(transfer_module, "_same_filesystem", lambda *_: False)
|
|
|
|
result = service.apply(plan["id"])
|
|
|
|
assert result == {
|
|
"plan_id": plan["id"],
|
|
"archived": 2,
|
|
"failed": 0,
|
|
"skipped": 0,
|
|
"state": "complete",
|
|
}
|
|
# The bytes moved: nothing is left in the library, everything is in the archive.
|
|
assert _contents(archive) == before
|
|
assert _contents(lib) == []
|
|
assert not folder.exists(), "an emptied album folder is not left behind"
|
|
for asset_id, digest in expected.items():
|
|
archived = archive / "rome" / _asset(sf, asset_id).archive_path.split("/")[-1]
|
|
assert sha256_file(archived) == digest
|
|
assert _states(service, plan["id"]) == [ArchiveState.COMPLETE] * 2
|
|
# No transfer temporaries survive either path.
|
|
assert not list((archive / "rome").glob(".archive-*"))
|
|
|
|
|
|
@pytest.mark.parametrize("same_filesystem", [True, False])
|
|
def test_the_manifest_on_the_medium_matches_the_archived_bytes(
|
|
tmp_path, monkeypatch, same_filesystem
|
|
):
|
|
"""The manifest is the medium's own record: it must be usable to verify the
|
|
archive with no database at all."""
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
_album(sf, lib)
|
|
location = _location(sf, config, archive)
|
|
service, plan = _plan(sf, config, location["id"])
|
|
if not same_filesystem:
|
|
monkeypatch.setattr(transfer_module, "_same_filesystem", lambda *_: False)
|
|
|
|
service.apply(plan["id"])
|
|
|
|
entries = read_manifest(archive / "rome")
|
|
assert len(entries) == 2
|
|
for entry in entries:
|
|
archived = archive / entry["archive_path"]
|
|
assert sha256_file(archived) == entry["sha256"]
|
|
assert entry["byte_size"] == archived.stat().st_size
|
|
assert entry["media_id"] == location["media_id"]
|
|
assert entry["plan_id"] == plan["id"] and entry["album"] == "rome"
|
|
|
|
|
|
def test_archived_assets_keep_their_identity_and_gain_their_new_location(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
folder, ids = _album(sf, lib)
|
|
location = _location(sf, config, archive)
|
|
service, plan = _plan(sf, config, location["id"])
|
|
|
|
service.apply(plan["id"])
|
|
|
|
with sf() as session:
|
|
for asset_id in ids:
|
|
asset = session.get(Asset, asset_id)
|
|
assert asset is not None, "archiving never deletes the record"
|
|
assert asset.current_path is None
|
|
assert asset.availability_state == "archived_online"
|
|
assert asset.archive_location_id == location["id"]
|
|
assert (archive / asset.archive_path).exists()
|
|
occurrences = session.scalars(
|
|
select(AssetPath).where(AssetPath.asset_id == asset_id)
|
|
).all()
|
|
active = [row for row in occurrences if row.valid_until is None]
|
|
assert [row.path for row in active] == [str(archive / asset.archive_path)]
|
|
closed = [row for row in occurrences if row.valid_until is not None]
|
|
assert [row.path for row in closed] == [str(folder / asset.archive_path.split("/")[-1])]
|
|
|
|
|
|
def test_applying_the_same_plan_again_changes_nothing(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
_album(sf, lib)
|
|
location = _location(sf, config, archive)
|
|
service, plan = _plan(sf, config, location["id"])
|
|
service.apply(plan["id"])
|
|
archived = _contents(archive)
|
|
manifest = read_manifest(archive / "rome")
|
|
|
|
again = service.apply(plan["id"])
|
|
|
|
assert again["skipped"] == 2 and again["archived"] == 0 and again["failed"] == 0
|
|
assert _contents(archive) == archived
|
|
assert read_manifest(archive / "rome") == manifest
|
|
|
|
|
|
# ── unexpected changes stop the item ─────────────────────────────────────────
|
|
|
|
|
|
def test_an_occupied_destination_is_never_overwritten(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
folder, _ = _album(sf, lib, names=("a.jpg",))
|
|
location = _location(sf, config, archive)
|
|
service, plan = _plan(sf, config, location["id"])
|
|
(archive / "rome").mkdir()
|
|
(archive / "rome" / "a.jpg").write_bytes(b"someone else's file")
|
|
|
|
result = service.apply(plan["id"])
|
|
|
|
assert result["failed"] == 1 and result["state"] == "failed"
|
|
assert (archive / "rome" / "a.jpg").read_bytes() == b"someone else's file"
|
|
assert (folder / "a.jpg").exists(), "the source must survive a refused transfer"
|
|
assert service.journal.operations(plan["id"])[0]["error_code"] == "destination_exists"
|
|
|
|
|
|
def test_a_source_edited_after_planning_is_neither_archived_nor_removed(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
folder, _ = _album(sf, lib, names=("a.jpg",))
|
|
location = _location(sf, config, archive)
|
|
service, plan = _plan(sf, config, location["id"])
|
|
|
|
(folder / "a.jpg").write_bytes(b"edited between approval and apply")
|
|
result = service.apply(plan["id"])
|
|
|
|
assert result["failed"] == 1
|
|
assert (folder / "a.jpg").read_bytes() == b"edited between approval and apply"
|
|
assert not (archive / "rome" / "a.jpg").exists()
|
|
assert service.journal.operations(plan["id"])[0]["error_code"] == "source_changed"
|
|
|
|
|
|
def test_a_copy_that_lands_with_the_wrong_bytes_is_not_published(tmp_path, monkeypatch):
|
|
"""The read-back hash — not the fact that a write returned — is what proves the
|
|
archive copy."""
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
folder, _ = _album(sf, lib, names=("a.jpg",))
|
|
location = _location(sf, config, archive)
|
|
service, plan = _plan(sf, config, location["id"])
|
|
monkeypatch.setattr(transfer_module, "_same_filesystem", lambda *_: False)
|
|
monkeypatch.setattr(
|
|
transfer_module.shutil,
|
|
"copyfileobj",
|
|
lambda src, dst, length=0: dst.write(b"corrupted in flight"),
|
|
)
|
|
|
|
result = service.apply(plan["id"])
|
|
|
|
assert result["failed"] == 1
|
|
assert service.journal.operations(plan["id"])[0]["error_code"] == "copy_mismatch"
|
|
assert (folder / "a.jpg").exists()
|
|
assert not (archive / "rome" / "a.jpg").exists()
|
|
assert not list((archive / "rome").glob(".archive-*")), "the failed copy is cleaned up"
|
|
|
|
|
|
def test_a_source_replaced_by_a_symlink_is_refused(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
folder, _ = _album(sf, lib, names=("a.jpg",))
|
|
location = _location(sf, config, archive)
|
|
service, plan = _plan(sf, config, location["id"])
|
|
elsewhere = tmp_path / "elsewhere.jpg"
|
|
elsewhere.write_bytes(b"not a library file")
|
|
(folder / "a.jpg").unlink()
|
|
(folder / "a.jpg").symlink_to(elsewhere)
|
|
|
|
result = service.apply(plan["id"])
|
|
|
|
assert result["failed"] == 1
|
|
assert elsewhere.exists() and (folder / "a.jpg").is_symlink()
|
|
assert not (archive / "rome" / "a.jpg").exists()
|
|
|
|
|
|
def test_one_failed_file_does_not_stop_the_rest_of_the_album(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
folder, _ = _album(sf, lib, names=("a.jpg", "b.jpg"))
|
|
location = _location(sf, config, archive)
|
|
service, plan = _plan(sf, config, location["id"])
|
|
(folder / "a.jpg").write_bytes(b"edited after approval")
|
|
|
|
result = service.apply(plan["id"])
|
|
|
|
assert (result["archived"], result["failed"]) == (1, 1)
|
|
assert (folder / "a.jpg").exists() and not (folder / "b.jpg").exists()
|
|
assert (archive / "rome" / "b.jpg").exists()
|
|
assert folder.exists(), "a folder that still holds an unarchived file stays"
|
|
|
|
|
|
def test_an_unresolved_transfer_blocks_the_next_preflight(tmp_path, monkeypatch):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
_album(sf, lib, "rome", names=("a.jpg",))
|
|
_album(sf, lib, "paris", names=("c.jpg",))
|
|
location = _location(sf, config, archive)
|
|
service, plan = _plan(sf, config, location["id"], ["rome"])
|
|
# Leave one item stuck mid-transfer, as a killed process would.
|
|
service.journal.begin(
|
|
service.journal.operations(plan["id"])[0]["id"], worker_id="w", fencing_token=1
|
|
)
|
|
|
|
report = ArchiveService(sf, config=config).preflight(location["id"], ["paris"])
|
|
|
|
assert report["state"] == "blocked"
|
|
assert "archive_pending" in {issue["code"] for issue in report["blockers"]}
|
|
|
|
|
|
# ── API surface ──────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_api_creates_a_plan_and_queues_it_on_the_archiver_lane(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
_album(sf, lib)
|
|
|
|
with TestClient(create_app(config)) as client:
|
|
location = client.post(
|
|
"/api/v1/archive-locations", json={"name": "external", "root": str(archive)}
|
|
).json()
|
|
token = client.post(
|
|
"/api/v1/archive-preflight", json={"location_id": location["id"]}
|
|
).json()["token"]
|
|
created = client.post(
|
|
"/api/v1/archive-plans", json={"location_id": location["id"], "token": token}
|
|
)
|
|
plan_id = created.json()["id"]
|
|
fetched = client.get(f"/api/v1/archive-plans/{plan_id}")
|
|
applied = client.post(f"/api/v1/archive-plans/{plan_id}/apply")
|
|
listed = client.get("/api/v1/archive-plans")
|
|
|
|
assert created.status_code == 201 and created.json()["asset_count"] == 2
|
|
assert fetched.status_code == 200 and len(fetched.json()["operations"]) == 2
|
|
assert applied.status_code == 200 and applied.json()["job"]["state"] == "queued"
|
|
assert applied.json()["job"]["lock_key"] == "archive"
|
|
assert [row["id"] for row in listed.json()["plans"]] == [plan_id]
|
|
# Queuing alone must not have touched a single file.
|
|
assert _contents(lib) and _contents(archive) == []
|
|
|
|
|
|
def test_api_refuses_a_stale_token_and_an_unknown_plan(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
_album(sf, lib)
|
|
location = _location(sf, config, archive)
|
|
|
|
with TestClient(create_app(config)) as client:
|
|
stale = client.post(
|
|
"/api/v1/archive-plans",
|
|
json={"location_id": location["id"], "token": "v1:not-the-real-token"},
|
|
)
|
|
unknown = client.get("/api/v1/archive-plans/nope")
|
|
unknown_apply = client.post("/api/v1/archive-plans/nope/apply")
|
|
|
|
assert stale.status_code == 409 and stale.json()["error"]["code"] == "stale_token"
|
|
assert unknown.status_code == 404 and unknown_apply.status_code == 404
|
|
|
|
|
|
def test_api_reports_recovery_state_for_an_interrupted_transfer(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
_album(sf, lib, names=("a.jpg",))
|
|
location = _location(sf, config, archive)
|
|
service, plan = _plan(sf, config, location["id"])
|
|
service.journal.begin(
|
|
service.journal.operations(plan["id"])[0]["id"], worker_id="w", fencing_token=1
|
|
)
|
|
|
|
with TestClient(create_app(config)) as client:
|
|
status = client.get("/api/v1/archive-recovery").json()
|
|
resolved = client.post("/api/v1/archive-recovery/resolve").json()
|
|
|
|
assert status["blocks_mutation"] is True
|
|
assert status["operations"][0]["classification"] == "resumable"
|
|
assert resolved == {"resumed": 1, "completed": 0, "manual": 0}
|
|
assert json.loads(json.dumps(resolved)) # plain JSON, nothing exotic
|