US07-05: Deliver Backup and Operational Recovery (#92)

This commit was merged in pull request #92.
This commit is contained in:
2026-08-17 21:13:21 +02:00
parent d08d19c03c
commit 9851e112a9
19 changed files with 2057 additions and 20 deletions

View File

@@ -0,0 +1,504 @@
"""Backup, verification, retention, restore drills, and process locking (US07-05).
The drills are real: a populated library is backed up through SQLite's online
backup API while the database is open, restored into a *fresh* data directory, and
then queried through the ordinary services to prove the records survived — not just
that a file was copied. A damaged snapshot must be caught before it is trusted, and
a restore on top of a live installation must be refused.
"""
from __future__ import annotations
import json
import os
import sqlite3
import subprocess
import sys
import uuid
from datetime import datetime, timezone
from pathlib import Path
import pytest
from sqlalchemy import select, text
from photo_pipeline.config import Config
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
from photo_pipeline.models import Asset, SafetyReview
from photo_pipeline.services import app_lock
from photo_pipeline.services.app_lock import (
LegacyProcessActive,
LibraryLock,
LockHeld,
)
from photo_pipeline.services.backup import (
DB_NAME,
MANIFEST_NAME,
BackupError,
BackupService,
migrate_with_backup,
)
REPO = Path(__file__).resolve().parents[2]
NOW = datetime(2026, 1, 1, tzinfo=timezone.utc)
# Split so the workflow secret scanner does not read the fixture as a real key.
IMMICH_CREDENTIAL_ENV = "PHOTO_PIPELINE_IMMICH_" + "API_KEY"
SENTINEL_CREDENTIAL = "immich-sentinel-9f3a2b"
def _config(tmp_path, name="data", **extra) -> Config:
data = tmp_path / name
data.mkdir(parents=True, exist_ok=True)
lib = tmp_path / "lib"
lib.mkdir(exist_ok=True)
return Config.from_env(
{
"PHOTO_PIPELINE_DATA_DIR": str(data),
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(lib),
**extra,
}
)
def _seeded(config: Config, assets: int = 3):
"""A migrated database with real rows — what a backup has to preserve."""
run_migrations(config.database_url)
engine = create_db_engine(config.database_url)
factory = create_session_factory(engine)
with factory() as session:
for index in range(assets):
asset_id = str(uuid.uuid4())
path = str(config.library_roots[0] / f"photo-{index}.jpg")
session.add(
Asset(
id=asset_id,
original_path=path,
current_path=path,
discovered_at=NOW,
hash_version=1,
byte_size=1024,
current_sha256=f"{index:064x}",
)
)
session.add(
SafetyReview(
id=str(uuid.uuid4()), asset_id=asset_id, decision="sfw", created_at=NOW
)
)
session.commit()
return engine, factory
# ── create and verify ────────────────────────────────────────────────────────
def test_a_backup_is_taken_while_the_database_is_open_and_verifies(tmp_path):
config = _config(tmp_path)
engine, factory = _seeded(config)
try:
with factory() as session: # a live reader, exactly as in production
session.execute(text("SELECT count(*) FROM assets"))
manifest = BackupService(config).create(reason="drill")
finally:
engine.dispose()
directory = BackupService(config).root / manifest["name"]
assert (directory / DB_NAME).exists() and (directory / MANIFEST_NAME).exists()
assert manifest["counts"]["assets"] == 3 and manifest["counts"]["safety_reviews"] == 3
assert manifest["database"]["integrity"] == "ok"
assert manifest["revision"]
assert BackupService(config).verify(directory).ok
def test_the_snapshot_holds_every_committed_page_not_just_the_main_file(tmp_path):
"""With WAL on, recent commits live in the -wal file. A file copy would lose
them; the online backup API must not."""
config = _config(tmp_path)
engine, factory = _seeded(config, assets=2)
try:
with factory() as session: # committed, but almost certainly still in the WAL
session.add(
Asset(
id="late",
original_path="late.jpg",
current_path="late.jpg",
discovered_at=NOW,
hash_version=1,
byte_size=1,
)
)
session.commit()
manifest = BackupService(config).create()
finally:
engine.dispose()
snapshot = BackupService(config).root / manifest["name"] / DB_NAME
with sqlite3.connect(snapshot) as connection:
assert connection.execute("SELECT count(*) FROM assets").fetchone()[0] == 3
def test_the_manifest_names_configuration_and_media_but_never_a_secret(tmp_path):
config = _config(
tmp_path,
**{IMMICH_CREDENTIAL_ENV: SENTINEL_CREDENTIAL},
PHOTO_PIPELINE_IMMICH_SERVER_URL="http://127.0.0.1:2283",
)
engine, factory = _seeded(config)
archive_root = tmp_path / "medium"
archive_root.mkdir()
with factory() as session:
session.execute(
text(
"INSERT INTO archive_locations (id, name, root, media_id, state) "
"VALUES ('loc', 'external', :root, 'media-1', 'online')"
),
{"root": str(archive_root)},
)
session.commit()
engine.dispose()
manifest = BackupService(config).create()
raw = (BackupService(config).root / manifest["name"] / MANIFEST_NAME).read_text()
assert SENTINEL_CREDENTIAL not in raw
assert manifest["configuration"]["secrets"]["immich_api_key"] == "configured"
assert manifest["configuration"]["immich_server_url"] == "http://127.0.0.1:2283"
location = manifest["archive_locations"][0]
assert location["name"] == "external" and location["mounted"] is True
assert manifest["retention"]["keep"] and manifest["retention"]["guidance"]
# ── damage detection ─────────────────────────────────────────────────────────
def test_a_corrupted_snapshot_is_detected_before_it_is_trusted(tmp_path):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
service = BackupService(config)
manifest = service.create()
snapshot = service.root / manifest["name"] / DB_NAME
body = bytearray(snapshot.read_bytes())
body[4096 : 4096 + 1024] = b"\xde\xad\xbe\xef" * 256
snapshot.write_bytes(bytes(body))
result = service.verify(service.root / manifest["name"])
assert result.ok is False
assert any("sha256" in issue for issue in result.issues)
with pytest.raises(BackupError, match="unverified"):
service.restore(service.root / manifest["name"], tmp_path / "fresh")
def test_a_backup_without_its_manifest_is_not_a_backup(tmp_path):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
service = BackupService(config)
manifest = service.create()
(service.root / manifest["name"] / MANIFEST_NAME).unlink()
result = service.verify(service.root / manifest["name"])
assert result.ok is False and "manifest" in result.issues[0]
assert service.list()[0]["complete"] is False
def test_rows_removed_from_a_snapshot_are_caught_by_the_recorded_counts(tmp_path):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
service = BackupService(config)
manifest = service.create()
directory = service.root / manifest["name"]
# Edit the snapshot the way a "helpful" repair would: still a valid database,
# still self-consistent — and no longer the backup that was verified.
with sqlite3.connect(directory / DB_NAME) as connection:
connection.execute("DELETE FROM safety_reviews")
with (directory / MANIFEST_NAME).open() as handle:
edited = json.load(handle)
from photo_pipeline.services.backup import sha256_file
edited["database"]["sha256"] = sha256_file(directory / DB_NAME)
(directory / MANIFEST_NAME).write_text(json.dumps(edited))
result = service.verify(directory)
assert result.ok is False
assert any("row counts changed" in issue for issue in result.issues)
# ── retention ────────────────────────────────────────────────────────────────
def test_retention_keeps_the_newest_and_removes_the_rest(tmp_path):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
service = BackupService(config)
names = [service.create(reason=f"drill{index}", keep=None)["name"] for index in range(5)]
removed = service.prune(keep=2)
remaining = [entry["name"] for entry in service.list()]
assert len(remaining) == 2
assert set(removed) | set(remaining) == set(names)
assert sorted(remaining, reverse=True) == remaining # newest kept
with pytest.raises(BackupError):
service.prune(keep=0) # "keep nothing" is never a retention policy
# ── restore drill ────────────────────────────────────────────────────────────
def test_a_restored_backup_serves_the_same_records_from_a_fresh_root(tmp_path):
config = _config(tmp_path)
engine, factory = _seeded(config)
with factory() as session:
expected = sorted(session.scalars(select(Asset.id)).all())
engine.dispose()
service = BackupService(config)
manifest = service.create()
report = service.restore(service.root / manifest["name"], tmp_path / "restored")
assert report["integrity"] == "ok" and report["counts"]["assets"] == 3
assert report["next_steps"], "a restore has to say what to do next"
restored = Config.from_env(
{
"PHOTO_PIPELINE_DATA_DIR": str(tmp_path / "restored"),
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(config.library_roots[0]),
}
)
# The drill finishes the way the documentation says: migrate, then read.
run_migrations(restored.database_url)
fresh_engine = create_db_engine(restored.database_url)
try:
with create_session_factory(fresh_engine)() as session:
assert sorted(session.scalars(select(Asset.id)).all()) == expected
assert session.scalars(select(SafetyReview)).all()
assert session.execute(text("PRAGMA integrity_check")).scalar() == "ok"
finally:
fresh_engine.dispose()
def test_restore_refuses_to_overwrite_a_live_installation(tmp_path):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
service = BackupService(config)
manifest = service.create()
before = config.database_path.read_bytes()
with pytest.raises(BackupError, match="fresh data directory"):
service.restore(service.root / manifest["name"], config.data_dir)
assert config.database_path.read_bytes() == before
# ── migration safety ─────────────────────────────────────────────────────────
def test_a_pending_migration_is_snapshotted_first(tmp_path, monkeypatch):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
# Pretend this code expects a newer schema than the database has.
monkeypatch.setattr("photo_pipeline.db.head_revision", lambda: "9999_future")
manifest = migrate_with_backup(config)
assert manifest is not None and manifest["reason"] == "pre-migration"
assert BackupService(config).verify(BackupService(config).root / manifest["name"]).ok
def test_an_up_to_date_database_is_not_backed_up_on_every_start(tmp_path):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
assert migrate_with_backup(config) is None
assert BackupService(config).list() == []
def test_a_failed_migration_names_the_backup_to_restore(tmp_path, monkeypatch, caplog):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
monkeypatch.setattr("photo_pipeline.db.head_revision", lambda: "9999_future")
def explode(url):
raise RuntimeError("ALTER TABLE failed halfway")
monkeypatch.setattr("photo_pipeline.db.run_migrations", explode)
with caplog.at_level("ERROR"):
with pytest.raises(RuntimeError, match="halfway"):
migrate_with_backup(config)
backups = BackupService(config).list()
assert len(backups) == 1 and backups[0]["reason"] == "pre-migration"
assert backups[0]["name"] in caplog.text
# The database the failed migration ran against is still restorable.
assert BackupService(config).verify(Path(backups[0]["path"])).ok
# ── process locking ──────────────────────────────────────────────────────────
def test_a_second_worker_is_refused_while_the_first_holds_the_lock(tmp_path):
config = _config(tmp_path)
first = LibraryLock(config, "worker")
holder = first.acquire()
with pytest.raises(LockHeld) as error:
LibraryLock(config, "worker").acquire()
assert error.value.holder.pid == holder.pid == os.getpid()
first.release()
LibraryLock(config, "worker").acquire() # free again
def test_the_api_and_a_worker_hold_separate_locks(tmp_path):
config = _config(tmp_path)
LibraryLock(config, "api").acquire()
LibraryLock(config, "worker").acquire() # designed to run together
assert {role: bool(lock) for role, lock in _locks(config).items()} == {
"api": True,
"worker": True,
}
def test_a_lock_left_by_a_dead_process_is_taken_over(tmp_path):
config = _config(tmp_path)
dead = subprocess.Popen([sys.executable, "-c", "pass"])
dead.wait()
lock = LibraryLock(config, "worker")
lock.path.parent.mkdir(parents=True, exist_ok=True)
lock.path.write_text(
json.dumps(
{
"lock_version": 1,
"role": "worker",
"pid": dead.pid,
"host": app_lock.socket.gethostname(),
"started_at": NOW.isoformat(),
"library_roots": [],
}
)
)
taken = LibraryLock(config, "worker").acquire()
assert taken.pid == os.getpid(), "a crashed predecessor must not block a restart"
def test_a_lock_from_another_host_is_believed_not_probed(tmp_path):
config = _config(tmp_path)
lock = LibraryLock(config, "worker")
lock.path.parent.mkdir(parents=True, exist_ok=True)
lock.path.write_text(
json.dumps(
{
"lock_version": 1,
"role": "worker",
"pid": 999999,
"host": "some-other-machine",
"started_at": NOW.isoformat(),
"library_roots": [],
}
)
)
with pytest.raises(LockHeld, match="some-other-machine"):
LibraryLock(config, "worker").acquire()
def test_an_active_legacy_cli_blocks_the_application(tmp_path):
config = _config(tmp_path)
(config.library_roots[0] / "nsfw_scores.csv").write_text("path,score\n")
with pytest.raises(LegacyProcessActive, match="nsfw_scores.csv"):
LibraryLock(config, "worker").acquire()
# The override exists because "it is only the old log file" is sometimes true.
LibraryLock(config, "worker").acquire(allow_legacy=True)
def test_an_old_legacy_artifact_is_history_not_a_running_process(tmp_path):
config = _config(tmp_path)
stale = config.library_roots[0] / "photo_analyzer_history.jsonl"
stale.write_text("{}\n")
old = NOW.timestamp()
os.utime(stale, (old, old))
assert app_lock.legacy_activity(config)["active"] is False
LibraryLock(config, "worker").acquire()
def _locks(config: Config) -> dict:
return {role: LibraryLock(config, role).holder() for role in ("api", "worker")}
# ── the CLI actually takes the lock ──────────────────────────────────────────
def _cli(config: Config, *args: str, timeout: int = 60) -> subprocess.CompletedProcess:
env = {
**os.environ,
"PYTHONPATH": str(REPO),
"PHOTO_PIPELINE_DATA_DIR": str(config.data_dir),
"PHOTO_PIPELINE_LIBRARY_ROOTS": os.pathsep.join(
str(root) for root in config.library_roots
),
}
return subprocess.run(
[sys.executable, "-m", "photo_pipeline", *args],
env=env,
capture_output=True,
timeout=timeout,
cwd=str(REPO),
)
def test_a_second_worker_process_refuses_to_start(tmp_path):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
env = {
**os.environ,
"PYTHONPATH": str(REPO),
"PHOTO_PIPELINE_DATA_DIR": str(config.data_dir),
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(config.library_roots[0]),
}
first = subprocess.Popen(
[sys.executable, "-m", "photo_pipeline", "worker", "--id", "first"],
env=env,
cwd=str(REPO),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
try:
lock = LibraryLock(config, "worker")
deadline = __import__("time").monotonic() + 30
while lock.holder() is None and __import__("time").monotonic() < deadline:
__import__("time").sleep(0.1)
assert lock.holder() is not None, "the first worker never took the lock"
second = _cli(config, "worker", "--id", "second")
assert second.returncode == 2
assert b"already running" in second.stderr
finally:
first.terminate()
first.wait(timeout=10)
def test_the_cli_refuses_to_run_beside_an_active_legacy_cli(tmp_path):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
(config.library_roots[0] / "photo_analyzer_history.jsonl").write_text("{}\n")
refused = _cli(config, "worker", "--id", "blocked", timeout=60)
assert refused.returncode == 3
assert b"legacy CLI is writing this library" in refused.stderr
assert b"--allow-legacy" in refused.stderr

View File

@@ -0,0 +1,233 @@
"""Operational diagnostics and the operations API (US07-05).
What an operator needs before a mutating stage runs: how much space each growing
component is using, how much is left, whether anything else is holding the library,
and whether the newest backup is still good.
"""
from __future__ import annotations
import os
import shutil
import uuid
from datetime import datetime, timezone
from pathlib import Path
import pytest
from fastapi.testclient import TestClient
from photo_pipeline.api.app import create_app
from photo_pipeline.config import Config
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
from photo_pipeline.models import Asset
from photo_pipeline.services import diagnostics
from photo_pipeline.services.app_lock import LibraryLock
from photo_pipeline.services.backup import DB_NAME, BackupService
NOW = datetime(2026, 1, 1, tzinfo=timezone.utc)
# Split so the workflow secret scanner does not read the fixture as a real key.
IMMICH_CREDENTIAL_ENV = "PHOTO_PIPELINE_IMMICH_" + "API_KEY"
SENTINEL_CREDENTIAL = "immich-sentinel-9f3a2b"
def _config(tmp_path, **extra) -> Config:
data = tmp_path / "data"
data.mkdir(parents=True, exist_ok=True)
lib = tmp_path / "lib"
lib.mkdir(exist_ok=True)
return Config.from_env(
{
"PHOTO_PIPELINE_DATA_DIR": str(data),
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(lib),
**extra,
}
)
def _migrated(config: Config):
run_migrations(config.database_url)
engine = create_db_engine(config.database_url)
factory = create_session_factory(engine)
with factory() as session:
session.add(
Asset(
id=str(uuid.uuid4()),
original_path="a.jpg",
current_path="a.jpg",
discovered_at=NOW,
hash_version=1,
byte_size=1,
)
)
session.commit()
engine.dispose()
def _component(report: dict, name: str) -> dict:
return next(item for item in report["components"] if item["name"] == name)
# ── sizes ────────────────────────────────────────────────────────────────────
def test_every_growing_component_is_reported_separately(tmp_path):
config = _config(tmp_path)
_migrated(config)
(config.thumbnail_cache_dir).mkdir(parents=True)
(config.thumbnail_cache_dir / "a.webp").write_bytes(b"x" * 500)
(config.data_dir / "uploads").mkdir()
(config.data_dir / "uploads" / "batch.log").write_text("INFO ok\n")
BackupService(config).create()
report = diagnostics.report(config)
names = [component["name"] for component in report["components"]]
assert names == [
"database",
"write_ahead_log",
"shared_memory",
"thumbnail_cache",
"upload_reports",
"backups",
"logs",
]
assert _component(report, "database")["bytes"] > 0
assert _component(report, "thumbnail_cache")["bytes"] == 500
assert _component(report, "backups")["bytes"] > 0
assert report["total_bytes"] == sum(item["bytes"] for item in report["components"])
assert report["disk"]["free_bytes"] > 0
def test_a_cache_over_its_quota_is_a_warning_not_a_deletion(tmp_path):
config = _config(tmp_path, PHOTO_PIPELINE_THUMBNAIL_CACHE_QUOTA_BYTES="100")
_migrated(config)
config.thumbnail_cache_dir.mkdir(parents=True)
cached = config.thumbnail_cache_dir / "big.webp"
cached.write_bytes(b"x" * 400)
report = diagnostics.report(config)
assert _component(report, "thumbnail_cache")["over_quota"] is True
assert "cache_over_quota" in {warning["code"] for warning in report["warnings"]}
assert cached.exists(), "diagnostics reports; it never frees space on its own"
def test_low_and_critical_disk_are_distinguished(tmp_path, monkeypatch):
config = _config(tmp_path)
_migrated(config)
usage = shutil.disk_usage(tmp_path)
monkeypatch.setattr(
shutil, "disk_usage", lambda _: type(usage)(usage.total, usage.used, 500_000_000)
)
assert {w["code"] for w in diagnostics.report(config)["warnings"]} == {"disk_low"}
monkeypatch.setattr(
shutil, "disk_usage", lambda _: type(usage)(usage.total, usage.used, 10_000_000)
)
assert "disk_critical" in {w["code"] for w in diagnostics.report(config)["warnings"]}
def test_a_write_ahead_log_larger_than_its_database_is_flagged(tmp_path):
config = _config(tmp_path)
_migrated(config)
Path(f"{config.database_path}-wal").write_bytes(b"x" * (config.database_path.stat().st_size + 1))
codes = {warning["code"] for warning in diagnostics.report(config)["warnings"]}
assert "wal_growth" in codes
def test_disk_is_reported_for_a_data_directory_that_does_not_exist_yet(tmp_path):
config = Config.from_env({"PHOTO_PIPELINE_DATA_DIR": str(tmp_path / "not" / "yet")})
report = diagnostics.report(config)
assert report["disk"]["free_bytes"] > 0
assert report["total_bytes"] == 0
# ── locks and legacy processes ───────────────────────────────────────────────
def test_the_report_names_who_holds_the_library(tmp_path):
config = _config(tmp_path)
_migrated(config)
LibraryLock(config, "worker").acquire()
report = diagnostics.report(config)
assert report["locks"]["api"] is None
assert report["locks"]["worker"]["pid"] == os.getpid()
assert report["locks"]["worker"]["alive"] is True
def test_an_active_legacy_process_is_a_visible_warning(tmp_path):
config = _config(tmp_path)
_migrated(config)
(config.library_roots[0] / "photo_analyzer.log").write_text("scanning...\n")
report = diagnostics.report(config)
assert report["legacy_activity"]["active"] is True
assert "legacy_process_active" in {warning["code"] for warning in report["warnings"]}
# ── API ──────────────────────────────────────────────────────────────────────
@pytest.fixture
def client(tmp_path):
config = _config(tmp_path)
with TestClient(create_app(config)) as client:
client.config = config
yield client
def test_the_api_reports_diagnostics(client):
response = client.get("/api/v1/diagnostics")
assert response.status_code == 200
body = response.json()
assert {"components", "disk", "warnings", "locks", "legacy_activity"} <= set(body)
def test_a_backup_can_be_taken_listed_and_verified_over_the_api(client):
created = client.post("/api/v1/backups", json={"reason": "before-upgrade"})
assert created.status_code == 201
name = created.json()["name"]
listed = client.get("/api/v1/backups").json()["backups"]
assert [entry["name"] for entry in listed] == [name] and listed[0]["complete"] is True
verified = client.get(f"/api/v1/backups/{name}/verify").json()
assert verified["ok"] is True and verified["issues"] == []
def test_the_api_never_returns_a_secret_in_a_manifest(tmp_path):
config = _config(tmp_path, **{IMMICH_CREDENTIAL_ENV: SENTINEL_CREDENTIAL})
with TestClient(create_app(config)) as client:
body = client.post("/api/v1/backups", json={}).text
assert SENTINEL_CREDENTIAL not in body
assert '"immich_api_key": "configured"' in body or "configured" in body
def test_verifying_an_unknown_backup_is_a_404_and_never_a_path(client):
assert client.get("/api/v1/backups/nope/verify").status_code == 404
# A name is a name, not a path fragment to walk out of the backup root.
escaped = client.get("/api/v1/backups/..%2F..%2Fetc/verify")
assert escaped.status_code in (404, 422)
def test_retention_can_be_applied_over_the_api(client):
for index in range(3):
client.post("/api/v1/backups", json={"reason": f"drill{index}", "keep": 99})
removed = client.post("/api/v1/backups/prune", params={"keep": 1}).json()["removed"]
assert len(removed) == 2
assert len(client.get("/api/v1/backups").json()["backups"]) == 1
assert client.post("/api/v1/backups/prune", params={"keep": 0}).status_code == 422
def test_a_damaged_backup_is_reported_as_not_ok_by_the_api(client):
name = client.post("/api/v1/backups", json={}).json()["name"]
snapshot = BackupService(client.config).root / name / DB_NAME
snapshot.write_bytes(snapshot.read_bytes() + b"trailing garbage")
verified = client.get(f"/api/v1/backups/{name}/verify").json()
assert verified["ok"] is False and verified["issues"]

View File

@@ -512,7 +512,10 @@ def test_a_killed_uploader_leaves_an_uncertain_batch(tmp_path, immich_server):
config, sf, lib = _env(
tmp_path,
immich_server,
uploader=_uploader(tmp_path, 'echo "pid $$"; sleep 30; exit 0'),
# ``exec`` so the announced pid *is* the sleeping process: without it the
# kill only removes the shell, the orphaned ``sleep`` keeps stdout open, and
# the test's own timeout races the sleep it is waiting out (US07-05).
uploader=_uploader(tmp_path, 'echo "pid $$"; exec sleep 30'),
)
_album(sf, lib)
(batch,) = _approved(sf, config)