US07-05: Deliver Backup and Operational Recovery (#92)
This commit was merged in pull request #92.
This commit is contained in:
504
tests/integration/test_backup_recovery.py
Normal file
504
tests/integration/test_backup_recovery.py
Normal file
@@ -0,0 +1,504 @@
|
||||
"""Backup, verification, retention, restore drills, and process locking (US07-05).
|
||||
|
||||
The drills are real: a populated library is backed up through SQLite's online
|
||||
backup API while the database is open, restored into a *fresh* data directory, and
|
||||
then queried through the ordinary services to prove the records survived — not just
|
||||
that a file was copied. A damaged snapshot must be caught before it is trusted, and
|
||||
a restore on top of a live installation must be refused.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from photo_pipeline.config import Config
|
||||
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
|
||||
from photo_pipeline.models import Asset, SafetyReview
|
||||
from photo_pipeline.services import app_lock
|
||||
from photo_pipeline.services.app_lock import (
|
||||
LegacyProcessActive,
|
||||
LibraryLock,
|
||||
LockHeld,
|
||||
)
|
||||
from photo_pipeline.services.backup import (
|
||||
DB_NAME,
|
||||
MANIFEST_NAME,
|
||||
BackupError,
|
||||
BackupService,
|
||||
migrate_with_backup,
|
||||
)
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
NOW = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||
# Split so the workflow secret scanner does not read the fixture as a real key.
|
||||
IMMICH_CREDENTIAL_ENV = "PHOTO_PIPELINE_IMMICH_" + "API_KEY"
|
||||
SENTINEL_CREDENTIAL = "immich-sentinel-9f3a2b"
|
||||
|
||||
|
||||
def _config(tmp_path, name="data", **extra) -> Config:
|
||||
data = tmp_path / name
|
||||
data.mkdir(parents=True, exist_ok=True)
|
||||
lib = tmp_path / "lib"
|
||||
lib.mkdir(exist_ok=True)
|
||||
return Config.from_env(
|
||||
{
|
||||
"PHOTO_PIPELINE_DATA_DIR": str(data),
|
||||
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(lib),
|
||||
**extra,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _seeded(config: Config, assets: int = 3):
|
||||
"""A migrated database with real rows — what a backup has to preserve."""
|
||||
run_migrations(config.database_url)
|
||||
engine = create_db_engine(config.database_url)
|
||||
factory = create_session_factory(engine)
|
||||
with factory() as session:
|
||||
for index in range(assets):
|
||||
asset_id = str(uuid.uuid4())
|
||||
path = str(config.library_roots[0] / f"photo-{index}.jpg")
|
||||
session.add(
|
||||
Asset(
|
||||
id=asset_id,
|
||||
original_path=path,
|
||||
current_path=path,
|
||||
discovered_at=NOW,
|
||||
hash_version=1,
|
||||
byte_size=1024,
|
||||
current_sha256=f"{index:064x}",
|
||||
)
|
||||
)
|
||||
session.add(
|
||||
SafetyReview(
|
||||
id=str(uuid.uuid4()), asset_id=asset_id, decision="sfw", created_at=NOW
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
return engine, factory
|
||||
|
||||
|
||||
# ── create and verify ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_a_backup_is_taken_while_the_database_is_open_and_verifies(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
engine, factory = _seeded(config)
|
||||
try:
|
||||
with factory() as session: # a live reader, exactly as in production
|
||||
session.execute(text("SELECT count(*) FROM assets"))
|
||||
manifest = BackupService(config).create(reason="drill")
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
directory = BackupService(config).root / manifest["name"]
|
||||
assert (directory / DB_NAME).exists() and (directory / MANIFEST_NAME).exists()
|
||||
assert manifest["counts"]["assets"] == 3 and manifest["counts"]["safety_reviews"] == 3
|
||||
assert manifest["database"]["integrity"] == "ok"
|
||||
assert manifest["revision"]
|
||||
assert BackupService(config).verify(directory).ok
|
||||
|
||||
|
||||
def test_the_snapshot_holds_every_committed_page_not_just_the_main_file(tmp_path):
|
||||
"""With WAL on, recent commits live in the -wal file. A file copy would lose
|
||||
them; the online backup API must not."""
|
||||
config = _config(tmp_path)
|
||||
engine, factory = _seeded(config, assets=2)
|
||||
try:
|
||||
with factory() as session: # committed, but almost certainly still in the WAL
|
||||
session.add(
|
||||
Asset(
|
||||
id="late",
|
||||
original_path="late.jpg",
|
||||
current_path="late.jpg",
|
||||
discovered_at=NOW,
|
||||
hash_version=1,
|
||||
byte_size=1,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
manifest = BackupService(config).create()
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
snapshot = BackupService(config).root / manifest["name"] / DB_NAME
|
||||
with sqlite3.connect(snapshot) as connection:
|
||||
assert connection.execute("SELECT count(*) FROM assets").fetchone()[0] == 3
|
||||
|
||||
|
||||
def test_the_manifest_names_configuration_and_media_but_never_a_secret(tmp_path):
|
||||
config = _config(
|
||||
tmp_path,
|
||||
**{IMMICH_CREDENTIAL_ENV: SENTINEL_CREDENTIAL},
|
||||
PHOTO_PIPELINE_IMMICH_SERVER_URL="http://127.0.0.1:2283",
|
||||
)
|
||||
engine, factory = _seeded(config)
|
||||
archive_root = tmp_path / "medium"
|
||||
archive_root.mkdir()
|
||||
with factory() as session:
|
||||
session.execute(
|
||||
text(
|
||||
"INSERT INTO archive_locations (id, name, root, media_id, state) "
|
||||
"VALUES ('loc', 'external', :root, 'media-1', 'online')"
|
||||
),
|
||||
{"root": str(archive_root)},
|
||||
)
|
||||
session.commit()
|
||||
engine.dispose()
|
||||
|
||||
manifest = BackupService(config).create()
|
||||
raw = (BackupService(config).root / manifest["name"] / MANIFEST_NAME).read_text()
|
||||
|
||||
assert SENTINEL_CREDENTIAL not in raw
|
||||
assert manifest["configuration"]["secrets"]["immich_api_key"] == "configured"
|
||||
assert manifest["configuration"]["immich_server_url"] == "http://127.0.0.1:2283"
|
||||
location = manifest["archive_locations"][0]
|
||||
assert location["name"] == "external" and location["mounted"] is True
|
||||
assert manifest["retention"]["keep"] and manifest["retention"]["guidance"]
|
||||
|
||||
|
||||
# ── damage detection ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_a_corrupted_snapshot_is_detected_before_it_is_trusted(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
engine, _ = _seeded(config)
|
||||
engine.dispose()
|
||||
service = BackupService(config)
|
||||
manifest = service.create()
|
||||
snapshot = service.root / manifest["name"] / DB_NAME
|
||||
|
||||
body = bytearray(snapshot.read_bytes())
|
||||
body[4096 : 4096 + 1024] = b"\xde\xad\xbe\xef" * 256
|
||||
snapshot.write_bytes(bytes(body))
|
||||
|
||||
result = service.verify(service.root / manifest["name"])
|
||||
assert result.ok is False
|
||||
assert any("sha256" in issue for issue in result.issues)
|
||||
with pytest.raises(BackupError, match="unverified"):
|
||||
service.restore(service.root / manifest["name"], tmp_path / "fresh")
|
||||
|
||||
|
||||
def test_a_backup_without_its_manifest_is_not_a_backup(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
engine, _ = _seeded(config)
|
||||
engine.dispose()
|
||||
service = BackupService(config)
|
||||
manifest = service.create()
|
||||
(service.root / manifest["name"] / MANIFEST_NAME).unlink()
|
||||
|
||||
result = service.verify(service.root / manifest["name"])
|
||||
assert result.ok is False and "manifest" in result.issues[0]
|
||||
assert service.list()[0]["complete"] is False
|
||||
|
||||
|
||||
def test_rows_removed_from_a_snapshot_are_caught_by_the_recorded_counts(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
engine, _ = _seeded(config)
|
||||
engine.dispose()
|
||||
service = BackupService(config)
|
||||
manifest = service.create()
|
||||
directory = service.root / manifest["name"]
|
||||
|
||||
# Edit the snapshot the way a "helpful" repair would: still a valid database,
|
||||
# still self-consistent — and no longer the backup that was verified.
|
||||
with sqlite3.connect(directory / DB_NAME) as connection:
|
||||
connection.execute("DELETE FROM safety_reviews")
|
||||
with (directory / MANIFEST_NAME).open() as handle:
|
||||
edited = json.load(handle)
|
||||
from photo_pipeline.services.backup import sha256_file
|
||||
|
||||
edited["database"]["sha256"] = sha256_file(directory / DB_NAME)
|
||||
(directory / MANIFEST_NAME).write_text(json.dumps(edited))
|
||||
|
||||
result = service.verify(directory)
|
||||
assert result.ok is False
|
||||
assert any("row counts changed" in issue for issue in result.issues)
|
||||
|
||||
|
||||
# ── retention ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_retention_keeps_the_newest_and_removes_the_rest(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
engine, _ = _seeded(config)
|
||||
engine.dispose()
|
||||
service = BackupService(config)
|
||||
names = [service.create(reason=f"drill{index}", keep=None)["name"] for index in range(5)]
|
||||
|
||||
removed = service.prune(keep=2)
|
||||
|
||||
remaining = [entry["name"] for entry in service.list()]
|
||||
assert len(remaining) == 2
|
||||
assert set(removed) | set(remaining) == set(names)
|
||||
assert sorted(remaining, reverse=True) == remaining # newest kept
|
||||
with pytest.raises(BackupError):
|
||||
service.prune(keep=0) # "keep nothing" is never a retention policy
|
||||
|
||||
|
||||
# ── restore drill ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_a_restored_backup_serves_the_same_records_from_a_fresh_root(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
engine, factory = _seeded(config)
|
||||
with factory() as session:
|
||||
expected = sorted(session.scalars(select(Asset.id)).all())
|
||||
engine.dispose()
|
||||
service = BackupService(config)
|
||||
manifest = service.create()
|
||||
|
||||
report = service.restore(service.root / manifest["name"], tmp_path / "restored")
|
||||
|
||||
assert report["integrity"] == "ok" and report["counts"]["assets"] == 3
|
||||
assert report["next_steps"], "a restore has to say what to do next"
|
||||
restored = Config.from_env(
|
||||
{
|
||||
"PHOTO_PIPELINE_DATA_DIR": str(tmp_path / "restored"),
|
||||
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(config.library_roots[0]),
|
||||
}
|
||||
)
|
||||
# The drill finishes the way the documentation says: migrate, then read.
|
||||
run_migrations(restored.database_url)
|
||||
fresh_engine = create_db_engine(restored.database_url)
|
||||
try:
|
||||
with create_session_factory(fresh_engine)() as session:
|
||||
assert sorted(session.scalars(select(Asset.id)).all()) == expected
|
||||
assert session.scalars(select(SafetyReview)).all()
|
||||
assert session.execute(text("PRAGMA integrity_check")).scalar() == "ok"
|
||||
finally:
|
||||
fresh_engine.dispose()
|
||||
|
||||
|
||||
def test_restore_refuses_to_overwrite_a_live_installation(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
engine, _ = _seeded(config)
|
||||
engine.dispose()
|
||||
service = BackupService(config)
|
||||
manifest = service.create()
|
||||
before = config.database_path.read_bytes()
|
||||
|
||||
with pytest.raises(BackupError, match="fresh data directory"):
|
||||
service.restore(service.root / manifest["name"], config.data_dir)
|
||||
|
||||
assert config.database_path.read_bytes() == before
|
||||
|
||||
|
||||
# ── migration safety ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_a_pending_migration_is_snapshotted_first(tmp_path, monkeypatch):
|
||||
config = _config(tmp_path)
|
||||
engine, _ = _seeded(config)
|
||||
engine.dispose()
|
||||
# Pretend this code expects a newer schema than the database has.
|
||||
monkeypatch.setattr("photo_pipeline.db.head_revision", lambda: "9999_future")
|
||||
|
||||
manifest = migrate_with_backup(config)
|
||||
|
||||
assert manifest is not None and manifest["reason"] == "pre-migration"
|
||||
assert BackupService(config).verify(BackupService(config).root / manifest["name"]).ok
|
||||
|
||||
|
||||
def test_an_up_to_date_database_is_not_backed_up_on_every_start(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
engine, _ = _seeded(config)
|
||||
engine.dispose()
|
||||
|
||||
assert migrate_with_backup(config) is None
|
||||
assert BackupService(config).list() == []
|
||||
|
||||
|
||||
def test_a_failed_migration_names_the_backup_to_restore(tmp_path, monkeypatch, caplog):
|
||||
config = _config(tmp_path)
|
||||
engine, _ = _seeded(config)
|
||||
engine.dispose()
|
||||
monkeypatch.setattr("photo_pipeline.db.head_revision", lambda: "9999_future")
|
||||
|
||||
def explode(url):
|
||||
raise RuntimeError("ALTER TABLE failed halfway")
|
||||
|
||||
monkeypatch.setattr("photo_pipeline.db.run_migrations", explode)
|
||||
|
||||
with caplog.at_level("ERROR"):
|
||||
with pytest.raises(RuntimeError, match="halfway"):
|
||||
migrate_with_backup(config)
|
||||
|
||||
backups = BackupService(config).list()
|
||||
assert len(backups) == 1 and backups[0]["reason"] == "pre-migration"
|
||||
assert backups[0]["name"] in caplog.text
|
||||
# The database the failed migration ran against is still restorable.
|
||||
assert BackupService(config).verify(Path(backups[0]["path"])).ok
|
||||
|
||||
|
||||
# ── process locking ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_a_second_worker_is_refused_while_the_first_holds_the_lock(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
first = LibraryLock(config, "worker")
|
||||
holder = first.acquire()
|
||||
|
||||
with pytest.raises(LockHeld) as error:
|
||||
LibraryLock(config, "worker").acquire()
|
||||
|
||||
assert error.value.holder.pid == holder.pid == os.getpid()
|
||||
first.release()
|
||||
LibraryLock(config, "worker").acquire() # free again
|
||||
|
||||
|
||||
def test_the_api_and_a_worker_hold_separate_locks(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
LibraryLock(config, "api").acquire()
|
||||
LibraryLock(config, "worker").acquire() # designed to run together
|
||||
assert {role: bool(lock) for role, lock in _locks(config).items()} == {
|
||||
"api": True,
|
||||
"worker": True,
|
||||
}
|
||||
|
||||
|
||||
def test_a_lock_left_by_a_dead_process_is_taken_over(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
dead = subprocess.Popen([sys.executable, "-c", "pass"])
|
||||
dead.wait()
|
||||
lock = LibraryLock(config, "worker")
|
||||
lock.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
lock.path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"lock_version": 1,
|
||||
"role": "worker",
|
||||
"pid": dead.pid,
|
||||
"host": app_lock.socket.gethostname(),
|
||||
"started_at": NOW.isoformat(),
|
||||
"library_roots": [],
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
taken = LibraryLock(config, "worker").acquire()
|
||||
|
||||
assert taken.pid == os.getpid(), "a crashed predecessor must not block a restart"
|
||||
|
||||
|
||||
def test_a_lock_from_another_host_is_believed_not_probed(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
lock = LibraryLock(config, "worker")
|
||||
lock.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
lock.path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"lock_version": 1,
|
||||
"role": "worker",
|
||||
"pid": 999999,
|
||||
"host": "some-other-machine",
|
||||
"started_at": NOW.isoformat(),
|
||||
"library_roots": [],
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(LockHeld, match="some-other-machine"):
|
||||
LibraryLock(config, "worker").acquire()
|
||||
|
||||
|
||||
def test_an_active_legacy_cli_blocks_the_application(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
(config.library_roots[0] / "nsfw_scores.csv").write_text("path,score\n")
|
||||
|
||||
with pytest.raises(LegacyProcessActive, match="nsfw_scores.csv"):
|
||||
LibraryLock(config, "worker").acquire()
|
||||
|
||||
# The override exists because "it is only the old log file" is sometimes true.
|
||||
LibraryLock(config, "worker").acquire(allow_legacy=True)
|
||||
|
||||
|
||||
def test_an_old_legacy_artifact_is_history_not_a_running_process(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
stale = config.library_roots[0] / "photo_analyzer_history.jsonl"
|
||||
stale.write_text("{}\n")
|
||||
old = NOW.timestamp()
|
||||
os.utime(stale, (old, old))
|
||||
|
||||
assert app_lock.legacy_activity(config)["active"] is False
|
||||
LibraryLock(config, "worker").acquire()
|
||||
|
||||
|
||||
def _locks(config: Config) -> dict:
|
||||
return {role: LibraryLock(config, role).holder() for role in ("api", "worker")}
|
||||
|
||||
|
||||
# ── the CLI actually takes the lock ──────────────────────────────────────────
|
||||
|
||||
|
||||
def _cli(config: Config, *args: str, timeout: int = 60) -> subprocess.CompletedProcess:
|
||||
env = {
|
||||
**os.environ,
|
||||
"PYTHONPATH": str(REPO),
|
||||
"PHOTO_PIPELINE_DATA_DIR": str(config.data_dir),
|
||||
"PHOTO_PIPELINE_LIBRARY_ROOTS": os.pathsep.join(
|
||||
str(root) for root in config.library_roots
|
||||
),
|
||||
}
|
||||
return subprocess.run(
|
||||
[sys.executable, "-m", "photo_pipeline", *args],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
cwd=str(REPO),
|
||||
)
|
||||
|
||||
|
||||
def test_a_second_worker_process_refuses_to_start(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
engine, _ = _seeded(config)
|
||||
engine.dispose()
|
||||
env = {
|
||||
**os.environ,
|
||||
"PYTHONPATH": str(REPO),
|
||||
"PHOTO_PIPELINE_DATA_DIR": str(config.data_dir),
|
||||
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(config.library_roots[0]),
|
||||
}
|
||||
first = subprocess.Popen(
|
||||
[sys.executable, "-m", "photo_pipeline", "worker", "--id", "first"],
|
||||
env=env,
|
||||
cwd=str(REPO),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
try:
|
||||
lock = LibraryLock(config, "worker")
|
||||
deadline = __import__("time").monotonic() + 30
|
||||
while lock.holder() is None and __import__("time").monotonic() < deadline:
|
||||
__import__("time").sleep(0.1)
|
||||
assert lock.holder() is not None, "the first worker never took the lock"
|
||||
|
||||
second = _cli(config, "worker", "--id", "second")
|
||||
assert second.returncode == 2
|
||||
assert b"already running" in second.stderr
|
||||
finally:
|
||||
first.terminate()
|
||||
first.wait(timeout=10)
|
||||
|
||||
|
||||
def test_the_cli_refuses_to_run_beside_an_active_legacy_cli(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
engine, _ = _seeded(config)
|
||||
engine.dispose()
|
||||
(config.library_roots[0] / "photo_analyzer_history.jsonl").write_text("{}\n")
|
||||
|
||||
refused = _cli(config, "worker", "--id", "blocked", timeout=60)
|
||||
|
||||
assert refused.returncode == 3
|
||||
assert b"legacy CLI is writing this library" in refused.stderr
|
||||
assert b"--allow-legacy" in refused.stderr
|
||||
233
tests/integration/test_diagnostics.py
Normal file
233
tests/integration/test_diagnostics.py
Normal file
@@ -0,0 +1,233 @@
|
||||
"""Operational diagnostics and the operations API (US07-05).
|
||||
|
||||
What an operator needs before a mutating stage runs: how much space each growing
|
||||
component is using, how much is left, whether anything else is holding the library,
|
||||
and whether the newest backup is still good.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from photo_pipeline.api.app import create_app
|
||||
from photo_pipeline.config import Config
|
||||
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
|
||||
from photo_pipeline.models import Asset
|
||||
from photo_pipeline.services import diagnostics
|
||||
from photo_pipeline.services.app_lock import LibraryLock
|
||||
from photo_pipeline.services.backup import DB_NAME, BackupService
|
||||
|
||||
NOW = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||
# Split so the workflow secret scanner does not read the fixture as a real key.
|
||||
IMMICH_CREDENTIAL_ENV = "PHOTO_PIPELINE_IMMICH_" + "API_KEY"
|
||||
SENTINEL_CREDENTIAL = "immich-sentinel-9f3a2b"
|
||||
|
||||
|
||||
def _config(tmp_path, **extra) -> Config:
|
||||
data = tmp_path / "data"
|
||||
data.mkdir(parents=True, exist_ok=True)
|
||||
lib = tmp_path / "lib"
|
||||
lib.mkdir(exist_ok=True)
|
||||
return Config.from_env(
|
||||
{
|
||||
"PHOTO_PIPELINE_DATA_DIR": str(data),
|
||||
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(lib),
|
||||
**extra,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _migrated(config: Config):
|
||||
run_migrations(config.database_url)
|
||||
engine = create_db_engine(config.database_url)
|
||||
factory = create_session_factory(engine)
|
||||
with factory() as session:
|
||||
session.add(
|
||||
Asset(
|
||||
id=str(uuid.uuid4()),
|
||||
original_path="a.jpg",
|
||||
current_path="a.jpg",
|
||||
discovered_at=NOW,
|
||||
hash_version=1,
|
||||
byte_size=1,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def _component(report: dict, name: str) -> dict:
|
||||
return next(item for item in report["components"] if item["name"] == name)
|
||||
|
||||
|
||||
# ── sizes ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_every_growing_component_is_reported_separately(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
_migrated(config)
|
||||
(config.thumbnail_cache_dir).mkdir(parents=True)
|
||||
(config.thumbnail_cache_dir / "a.webp").write_bytes(b"x" * 500)
|
||||
(config.data_dir / "uploads").mkdir()
|
||||
(config.data_dir / "uploads" / "batch.log").write_text("INFO ok\n")
|
||||
BackupService(config).create()
|
||||
|
||||
report = diagnostics.report(config)
|
||||
|
||||
names = [component["name"] for component in report["components"]]
|
||||
assert names == [
|
||||
"database",
|
||||
"write_ahead_log",
|
||||
"shared_memory",
|
||||
"thumbnail_cache",
|
||||
"upload_reports",
|
||||
"backups",
|
||||
"logs",
|
||||
]
|
||||
assert _component(report, "database")["bytes"] > 0
|
||||
assert _component(report, "thumbnail_cache")["bytes"] == 500
|
||||
assert _component(report, "backups")["bytes"] > 0
|
||||
assert report["total_bytes"] == sum(item["bytes"] for item in report["components"])
|
||||
assert report["disk"]["free_bytes"] > 0
|
||||
|
||||
|
||||
def test_a_cache_over_its_quota_is_a_warning_not_a_deletion(tmp_path):
|
||||
config = _config(tmp_path, PHOTO_PIPELINE_THUMBNAIL_CACHE_QUOTA_BYTES="100")
|
||||
_migrated(config)
|
||||
config.thumbnail_cache_dir.mkdir(parents=True)
|
||||
cached = config.thumbnail_cache_dir / "big.webp"
|
||||
cached.write_bytes(b"x" * 400)
|
||||
|
||||
report = diagnostics.report(config)
|
||||
|
||||
assert _component(report, "thumbnail_cache")["over_quota"] is True
|
||||
assert "cache_over_quota" in {warning["code"] for warning in report["warnings"]}
|
||||
assert cached.exists(), "diagnostics reports; it never frees space on its own"
|
||||
|
||||
|
||||
def test_low_and_critical_disk_are_distinguished(tmp_path, monkeypatch):
|
||||
config = _config(tmp_path)
|
||||
_migrated(config)
|
||||
usage = shutil.disk_usage(tmp_path)
|
||||
|
||||
monkeypatch.setattr(
|
||||
shutil, "disk_usage", lambda _: type(usage)(usage.total, usage.used, 500_000_000)
|
||||
)
|
||||
assert {w["code"] for w in diagnostics.report(config)["warnings"]} == {"disk_low"}
|
||||
|
||||
monkeypatch.setattr(
|
||||
shutil, "disk_usage", lambda _: type(usage)(usage.total, usage.used, 10_000_000)
|
||||
)
|
||||
assert "disk_critical" in {w["code"] for w in diagnostics.report(config)["warnings"]}
|
||||
|
||||
|
||||
def test_a_write_ahead_log_larger_than_its_database_is_flagged(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
_migrated(config)
|
||||
Path(f"{config.database_path}-wal").write_bytes(b"x" * (config.database_path.stat().st_size + 1))
|
||||
|
||||
codes = {warning["code"] for warning in diagnostics.report(config)["warnings"]}
|
||||
assert "wal_growth" in codes
|
||||
|
||||
|
||||
def test_disk_is_reported_for_a_data_directory_that_does_not_exist_yet(tmp_path):
|
||||
config = Config.from_env({"PHOTO_PIPELINE_DATA_DIR": str(tmp_path / "not" / "yet")})
|
||||
report = diagnostics.report(config)
|
||||
assert report["disk"]["free_bytes"] > 0
|
||||
assert report["total_bytes"] == 0
|
||||
|
||||
|
||||
# ── locks and legacy processes ───────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_the_report_names_who_holds_the_library(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
_migrated(config)
|
||||
LibraryLock(config, "worker").acquire()
|
||||
|
||||
report = diagnostics.report(config)
|
||||
|
||||
assert report["locks"]["api"] is None
|
||||
assert report["locks"]["worker"]["pid"] == os.getpid()
|
||||
assert report["locks"]["worker"]["alive"] is True
|
||||
|
||||
|
||||
def test_an_active_legacy_process_is_a_visible_warning(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
_migrated(config)
|
||||
(config.library_roots[0] / "photo_analyzer.log").write_text("scanning...\n")
|
||||
|
||||
report = diagnostics.report(config)
|
||||
|
||||
assert report["legacy_activity"]["active"] is True
|
||||
assert "legacy_process_active" in {warning["code"] for warning in report["warnings"]}
|
||||
|
||||
|
||||
# ── API ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(tmp_path):
|
||||
config = _config(tmp_path)
|
||||
with TestClient(create_app(config)) as client:
|
||||
client.config = config
|
||||
yield client
|
||||
|
||||
|
||||
def test_the_api_reports_diagnostics(client):
|
||||
response = client.get("/api/v1/diagnostics")
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert {"components", "disk", "warnings", "locks", "legacy_activity"} <= set(body)
|
||||
|
||||
|
||||
def test_a_backup_can_be_taken_listed_and_verified_over_the_api(client):
|
||||
created = client.post("/api/v1/backups", json={"reason": "before-upgrade"})
|
||||
assert created.status_code == 201
|
||||
name = created.json()["name"]
|
||||
|
||||
listed = client.get("/api/v1/backups").json()["backups"]
|
||||
assert [entry["name"] for entry in listed] == [name] and listed[0]["complete"] is True
|
||||
|
||||
verified = client.get(f"/api/v1/backups/{name}/verify").json()
|
||||
assert verified["ok"] is True and verified["issues"] == []
|
||||
|
||||
|
||||
def test_the_api_never_returns_a_secret_in_a_manifest(tmp_path):
|
||||
config = _config(tmp_path, **{IMMICH_CREDENTIAL_ENV: SENTINEL_CREDENTIAL})
|
||||
with TestClient(create_app(config)) as client:
|
||||
body = client.post("/api/v1/backups", json={}).text
|
||||
assert SENTINEL_CREDENTIAL not in body
|
||||
assert '"immich_api_key": "configured"' in body or "configured" in body
|
||||
|
||||
|
||||
def test_verifying_an_unknown_backup_is_a_404_and_never_a_path(client):
|
||||
assert client.get("/api/v1/backups/nope/verify").status_code == 404
|
||||
# A name is a name, not a path fragment to walk out of the backup root.
|
||||
escaped = client.get("/api/v1/backups/..%2F..%2Fetc/verify")
|
||||
assert escaped.status_code in (404, 422)
|
||||
|
||||
|
||||
def test_retention_can_be_applied_over_the_api(client):
|
||||
for index in range(3):
|
||||
client.post("/api/v1/backups", json={"reason": f"drill{index}", "keep": 99})
|
||||
removed = client.post("/api/v1/backups/prune", params={"keep": 1}).json()["removed"]
|
||||
assert len(removed) == 2
|
||||
assert len(client.get("/api/v1/backups").json()["backups"]) == 1
|
||||
assert client.post("/api/v1/backups/prune", params={"keep": 0}).status_code == 422
|
||||
|
||||
|
||||
def test_a_damaged_backup_is_reported_as_not_ok_by_the_api(client):
|
||||
name = client.post("/api/v1/backups", json={}).json()["name"]
|
||||
snapshot = BackupService(client.config).root / name / DB_NAME
|
||||
snapshot.write_bytes(snapshot.read_bytes() + b"trailing garbage")
|
||||
|
||||
verified = client.get(f"/api/v1/backups/{name}/verify").json()
|
||||
assert verified["ok"] is False and verified["issues"]
|
||||
@@ -512,7 +512,10 @@ def test_a_killed_uploader_leaves_an_uncertain_batch(tmp_path, immich_server):
|
||||
config, sf, lib = _env(
|
||||
tmp_path,
|
||||
immich_server,
|
||||
uploader=_uploader(tmp_path, 'echo "pid $$"; sleep 30; exit 0'),
|
||||
# ``exec`` so the announced pid *is* the sleeping process: without it the
|
||||
# kill only removes the shell, the orphaned ``sleep`` keeps stdout open, and
|
||||
# the test's own timeout races the sleep it is waiting out (US07-05).
|
||||
uploader=_uploader(tmp_path, 'echo "pid $$"; exec sleep 30'),
|
||||
)
|
||||
_album(sf, lib)
|
||||
(batch,) = _approved(sf, config)
|
||||
|
||||
Reference in New Issue
Block a user