314 lines
12 KiB
Python
314 lines
12 KiB
Python
"""Recovering interrupted archive transfers (US06-02).
|
|
|
|
The crash tests are real: a child process applies an archive plan and is killed by
|
|
the ``PHOTO_PIPELINE_FAULT_AFTER`` barrier at each persisted transition in turn —
|
|
including the moment immediately after an active source has been unlinked. The
|
|
parent then reopens the database and asserts the one invariant archiving exists to
|
|
uphold: **no verified file is ever lost, and no source is removed without a durable,
|
|
byte-identical archive copy.**
|
|
|
|
Both transfer paths are exercised: the same-filesystem atomic move and (with the
|
|
child forcing the device comparison) the cross-filesystem copy/verify/publish.
|
|
"""
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from sqlalchemy import select
|
|
|
|
from photo_pipeline.config import Config
|
|
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
|
|
from photo_pipeline.models import Asset, UploadBatch, UploadItem
|
|
from photo_pipeline.services.archive_journal import FORWARD, MANUAL, RESUMABLE, ArchiveState
|
|
from photo_pipeline.services.archive_transfer import (
|
|
MANIFEST_NAME,
|
|
ArchiveTransferService,
|
|
read_manifest,
|
|
)
|
|
from photo_pipeline.services.archives import ArchiveService
|
|
from photo_pipeline.services.hashing import sha256_file
|
|
|
|
pytestmark = pytest.mark.phase_f
|
|
|
|
REPO = Path(__file__).resolve().parents[2]
|
|
NOW = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
|
|
|
# Barriers, in the order the transfer persists them. ``source_removed`` is the one
|
|
# that matters most: the original is already gone at that point.
|
|
BARRIERS = [
|
|
ArchiveState.TRANSFERRING,
|
|
ArchiveState.VERIFIED,
|
|
ArchiveState.REMOVING,
|
|
"source_removed",
|
|
ArchiveState.COMPLETE,
|
|
]
|
|
|
|
# A child that applies the plan and dies at the configured barrier. ``force_copy``
|
|
# makes it take the cross-filesystem path without a second real volume.
|
|
APPLY_SCRIPT = """
|
|
import sys
|
|
sys.path.insert(0, {repo!r})
|
|
from photo_pipeline.config import Config
|
|
from photo_pipeline.db import create_db_engine, create_session_factory
|
|
from photo_pipeline.services import archive_transfer
|
|
|
|
db_url, data_dir, lib, plan_id, force_copy = sys.argv[1:6]
|
|
if force_copy == "1":
|
|
archive_transfer._same_filesystem = lambda *args: False
|
|
config = Config.from_env(
|
|
{{"PHOTO_PIPELINE_DATA_DIR": data_dir, "PHOTO_PIPELINE_LIBRARY_ROOTS": lib}}
|
|
)
|
|
sf = create_session_factory(create_db_engine(db_url))
|
|
archive_transfer.ArchiveTransferService(sf, config=config).apply(plan_id)
|
|
"""
|
|
|
|
|
|
# ── environment ──────────────────────────────────────────────────────────────
|
|
|
|
|
|
def _env(tmp_path):
|
|
(tmp_path / "data").mkdir(exist_ok=True)
|
|
lib = tmp_path / "lib"
|
|
lib.mkdir(exist_ok=True)
|
|
archive = tmp_path / "archive"
|
|
archive.mkdir(exist_ok=True)
|
|
config = Config.from_env(
|
|
{
|
|
"PHOTO_PIPELINE_DATA_DIR": str(tmp_path / "data"),
|
|
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(lib),
|
|
"PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES": "0",
|
|
}
|
|
)
|
|
run_migrations(config.database_url)
|
|
return config, create_session_factory(create_db_engine(config.database_url)), lib, archive
|
|
|
|
|
|
def _album(sf, lib, album="rome", names=("a.jpg", "b.jpg")):
|
|
folder = lib / album
|
|
folder.mkdir(parents=True, exist_ok=True)
|
|
with sf() as session:
|
|
batch_id = str(uuid.uuid4())
|
|
session.add(
|
|
UploadBatch(
|
|
id=batch_id,
|
|
album=album,
|
|
folder=str(folder),
|
|
album_name=album,
|
|
state="succeeded",
|
|
preflight_token="v1:test",
|
|
outcome_state="verified",
|
|
created_at=NOW,
|
|
)
|
|
)
|
|
for name in names:
|
|
path = folder / name
|
|
path.write_bytes(f"{album}/{name} content".encode() * 8)
|
|
asset_id = str(uuid.uuid4())
|
|
session.add(
|
|
Asset(
|
|
id=asset_id,
|
|
original_path=str(path),
|
|
current_path=str(path),
|
|
discovered_at=NOW,
|
|
hash_version=1,
|
|
byte_size=path.stat().st_size,
|
|
current_sha256=sha256_file(path),
|
|
)
|
|
)
|
|
session.add(
|
|
UploadItem(
|
|
batch_id=batch_id,
|
|
asset_id=asset_id,
|
|
path=str(path),
|
|
sha256=sha256_file(path),
|
|
sha1="0" * 40,
|
|
state="sent",
|
|
outcome="uploaded",
|
|
)
|
|
)
|
|
session.commit()
|
|
return folder
|
|
|
|
|
|
def _plan(sf, config, archive, albums=None):
|
|
location = ArchiveService(sf, config=config).register("external", str(archive))
|
|
token = ArchiveService(sf, config=config).preflight(location["id"], albums)["token"]
|
|
service = ArchiveTransferService(sf, config=config)
|
|
return service, service.create(location["id"], albums, token=token)
|
|
|
|
|
|
def _crash_during_apply(config, tmp_path, lib, plan, barrier, *, force_copy=False):
|
|
script = tmp_path / f"apply_{barrier}.py"
|
|
script.write_text(APPLY_SCRIPT.format(repo=str(REPO)))
|
|
env = dict(os.environ)
|
|
env["PHOTO_PIPELINE_FAULT_AFTER"] = barrier
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(script),
|
|
config.database_url,
|
|
str(tmp_path / "data"),
|
|
str(lib),
|
|
plan["id"],
|
|
"1" if force_copy else "0",
|
|
],
|
|
env=env,
|
|
capture_output=True,
|
|
)
|
|
assert result.returncode in (9, -9), (
|
|
f"child should have been killed at {barrier}, got {result.returncode}: "
|
|
f"{result.stderr.decode(errors='replace')[-400:]}"
|
|
)
|
|
|
|
|
|
def _contents(*roots):
|
|
return sorted(
|
|
path.read_bytes()
|
|
for root in roots
|
|
for path in root.rglob("*")
|
|
if path.is_file() and path.name != MANIFEST_NAME and not path.name.startswith(".")
|
|
)
|
|
|
|
|
|
def _reopen(config):
|
|
return create_session_factory(create_db_engine(config.database_url))
|
|
|
|
|
|
# ── fault injection at every persisted transition ────────────────────────────
|
|
|
|
|
|
@pytest.mark.parametrize("barrier", BARRIERS)
|
|
@pytest.mark.parametrize("force_copy", [False, True], ids=["move", "copy"])
|
|
def test_a_crash_at_every_transition_loses_nothing_and_recovers(tmp_path, barrier, force_copy):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
_album(sf, lib)
|
|
service, plan = _plan(sf, config, archive)
|
|
before = _contents(lib)
|
|
|
|
_crash_during_apply(config, tmp_path, lib, plan, barrier, force_copy=force_copy)
|
|
|
|
# Every file still exists somewhere: the crash may not have cost a single byte.
|
|
assert set(before) <= set(_contents(lib, archive)), f"content lost at {barrier}"
|
|
|
|
reopened = _reopen(config)
|
|
recovery = ArchiveTransferService(reopened, config=config)
|
|
# Whatever the crash interrupted is recoverable from evidence — never ambiguous.
|
|
# (A crash on the last barrier can land on a terminal item and leave none.)
|
|
verdicts = recovery.journal.classify_all()
|
|
assert all(v["classification"] in {RESUMABLE, FORWARD} for v in verdicts), verdicts
|
|
|
|
recovery.recover()
|
|
recovery.apply(plan["id"]) # finish whatever the crash never started
|
|
|
|
assert _contents(archive) == before
|
|
assert _contents(lib) == []
|
|
assert recovery.journal.blocks_mutation() is False
|
|
assert recovery.journal.plan_state(plan["id"]) == "complete"
|
|
|
|
|
|
@pytest.mark.parametrize("barrier", [ArchiveState.VERIFIED, "source_removed"])
|
|
def test_recovery_is_idempotent_across_repeated_restarts(tmp_path, barrier):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
_album(sf, lib)
|
|
service, plan = _plan(sf, config, archive)
|
|
_crash_during_apply(config, tmp_path, lib, plan, barrier)
|
|
|
|
reopened = _reopen(config)
|
|
recovery = ArchiveTransferService(reopened, config=config)
|
|
recovery.recover()
|
|
settled = (_contents(lib), _contents(archive), read_manifest(archive / "rome"))
|
|
|
|
for _ in range(2):
|
|
recovery.recover()
|
|
assert (_contents(lib), _contents(archive), read_manifest(archive / "rome")) == settled
|
|
|
|
|
|
def test_a_crash_before_anything_was_published_leaves_the_source_intact(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
folder = _album(sf, lib, names=("a.jpg",))
|
|
service, plan = _plan(sf, config, archive)
|
|
|
|
_crash_during_apply(config, tmp_path, lib, plan, ArchiveState.TRANSFERRING)
|
|
|
|
recovery = ArchiveTransferService(_reopen(config), config=config)
|
|
verdict = recovery.journal.classify_all()[0]
|
|
assert verdict["classification"] == RESUMABLE
|
|
assert (folder / "a.jpg").exists()
|
|
assert not (archive / "rome" / "a.jpg").exists()
|
|
|
|
assert recovery.recover() == {"resumed": 1, "completed": 0, "manual": 0}
|
|
assert recovery.journal.operations(plan["id"])[0]["journal_state"] == ArchiveState.PLANNED
|
|
|
|
|
|
def test_a_crash_after_the_source_was_removed_finishes_the_bookkeeping(tmp_path):
|
|
"""The dangerous window: the original is gone and the database still points at
|
|
it. Recovery must complete the record, never re-transfer or report loss."""
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
folder = _album(sf, lib, names=("a.jpg",))
|
|
service, plan = _plan(sf, config, archive)
|
|
expected = service.journal.operations(plan["id"])[0]["expected_sha256"]
|
|
|
|
_crash_during_apply(config, tmp_path, lib, plan, "source_removed")
|
|
|
|
reopened = _reopen(config)
|
|
recovery = ArchiveTransferService(reopened, config=config)
|
|
verdict = recovery.journal.classify_all()[0]
|
|
assert verdict["classification"] == FORWARD
|
|
assert not (folder / "a.jpg").exists()
|
|
assert sha256_file(archive / "rome" / "a.jpg") == expected
|
|
with reopened() as session:
|
|
stranded = session.scalar(select(Asset))
|
|
assert stranded.current_path is not None, "the crash happened before the DB update"
|
|
|
|
assert recovery.recover() == {"resumed": 0, "completed": 1, "manual": 0}
|
|
|
|
with reopened() as session:
|
|
asset = session.scalar(select(Asset))
|
|
assert asset.current_path is None
|
|
assert asset.availability_state == "archived_online"
|
|
assert asset.archive_path == "rome/a.jpg"
|
|
assert recovery.journal.operations(plan["id"])[0]["journal_state"] == ArchiveState.COMPLETE
|
|
assert len(read_manifest(archive / "rome")) == 1, "the manifest is not duplicated"
|
|
|
|
|
|
def test_an_archive_copy_that_changed_after_the_crash_blocks_for_a_human(tmp_path):
|
|
"""Wrong bytes at the destination can never justify deleting the original, and
|
|
are never overwritten either."""
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
folder = _album(sf, lib, names=("a.jpg",))
|
|
service, plan = _plan(sf, config, archive)
|
|
_crash_during_apply(config, tmp_path, lib, plan, ArchiveState.VERIFIED, force_copy=True)
|
|
(archive / "rome" / "a.jpg").write_bytes(b"tampered with while the app was down")
|
|
|
|
recovery = ArchiveTransferService(_reopen(config), config=config)
|
|
verdict = recovery.journal.classify_all()[0]
|
|
|
|
assert verdict["classification"] == MANUAL
|
|
assert recovery.recover() == {"resumed": 0, "completed": 0, "manual": 1}
|
|
assert (folder / "a.jpg").exists(), "the source is kept while the archive is unproven"
|
|
assert (archive / "rome" / "a.jpg").read_bytes() == b"tampered with while the app was down"
|
|
# And the unresolved item keeps blocking further archiving.
|
|
assert recovery.journal.blocks_mutation() is True
|
|
report = ArchiveService(_reopen(config), config=config).preflight(plan["location_id"])
|
|
assert "archive_pending" in {issue["code"] for issue in report["blockers"]}
|
|
|
|
|
|
def test_a_verified_archive_whose_copy_vanished_is_never_reported_as_archived(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
folder = _album(sf, lib, names=("a.jpg",))
|
|
service, plan = _plan(sf, config, archive)
|
|
_crash_during_apply(config, tmp_path, lib, plan, ArchiveState.REMOVING, force_copy=True)
|
|
(archive / "rome" / "a.jpg").unlink() # the medium lost it
|
|
|
|
recovery = ArchiveTransferService(_reopen(config), config=config)
|
|
|
|
assert recovery.journal.classify_all()[0]["classification"] == MANUAL
|
|
assert recovery.recover()["manual"] == 1
|
|
assert (folder / "a.jpg").exists()
|
|
assert recovery.journal.operations(plan["id"])[0]["journal_state"] != ArchiveState.COMPLETE
|