528 lines
21 KiB
Python
528 lines
21 KiB
Python
"""The fault matrix (US07-04, concept §18 "crash/fault-injection tests").
|
|
|
|
Process death at each persisted transition lives in tests/e2e/test_crash_recovery.py
|
|
and the per-stage recovery suites. This file covers the *environmental* faults —
|
|
the ones that are not a crash but are just as good at corrupting a library if the
|
|
code guesses:
|
|
|
|
disk full · read-only path · database busy · database corruption ·
|
|
network failure · malformed provider output · GPU exhaustion ·
|
|
subprocess hang · missing external tool
|
|
|
|
Every case asserts the same shape of outcome: the operation fails visibly, the
|
|
failure names what happened, and nothing irreversible was done on the way — no
|
|
source removed, no metadata marked verified, no decision invented.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import errno
|
|
import os
|
|
import sqlite3
|
|
import stat
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from contextlib import contextmanager
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pytest
|
|
from PIL import Image
|
|
from sqlalchemy import select, text
|
|
from sqlalchemy.exc import DatabaseError, OperationalError
|
|
|
|
from photo_pipeline import faults
|
|
from photo_pipeline.config import Config
|
|
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
|
|
from photo_pipeline.jobs.worker import Worker
|
|
from photo_pipeline.models import (
|
|
AlbumProposal,
|
|
AnalysisResult,
|
|
Asset,
|
|
ExifProjection,
|
|
SafetyReview,
|
|
UploadBatch,
|
|
UploadItem,
|
|
)
|
|
from photo_pipeline.services import archive_transfer, exif_checkpoint, hashing, rename_apply
|
|
from photo_pipeline.services.analysis import AnalysisService
|
|
from photo_pipeline.services.archive_transfer import ArchiveTransferService
|
|
from photo_pipeline.services.archives import ArchiveService
|
|
from photo_pipeline.services.jobs import ItemState, JobService
|
|
from photo_pipeline.services.rename_apply import ApplyError, RenameApplyService
|
|
from photo_pipeline.services.rename_journal import JournalState, RenameJournal
|
|
from photo_pipeline.services.renames import RenameService
|
|
from photo_pipeline.services.safety import SafetyService
|
|
from photo_pipeline.services.uploads import UploadService
|
|
|
|
NOW = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
|
|
|
|
|
# ── environment ──────────────────────────────────────────────────────────────
|
|
|
|
|
|
def _env(tmp_path, **extra):
|
|
(tmp_path / "data").mkdir(exist_ok=True)
|
|
lib = tmp_path / "lib"
|
|
lib.mkdir(exist_ok=True)
|
|
config = Config.from_env(
|
|
{
|
|
"PHOTO_PIPELINE_DATA_DIR": str(tmp_path / "data"),
|
|
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(lib),
|
|
"PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES": "0",
|
|
**extra,
|
|
}
|
|
)
|
|
run_migrations(config.database_url)
|
|
return config, create_session_factory(create_db_engine(config.database_url)), lib
|
|
|
|
|
|
def image(path: Path, seed: int = 1) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
pixels = np.random.default_rng(seed).integers(0, 256, (64, 96, 3), dtype=np.uint8)
|
|
Image.fromarray(pixels).save(path, quality=90)
|
|
|
|
|
|
def register(sf, path: Path) -> str:
|
|
asset_id = str(uuid.uuid4())
|
|
with sf() as session:
|
|
session.add(
|
|
Asset(
|
|
id=asset_id,
|
|
original_path=str(path),
|
|
current_path=str(path),
|
|
discovered_at=NOW,
|
|
hash_version=1,
|
|
byte_size=path.stat().st_size,
|
|
current_sha256=hashing.sha256_file(path),
|
|
)
|
|
)
|
|
session.commit()
|
|
return asset_id
|
|
|
|
|
|
def uploaded_album(sf, lib, album="rome", names=("a.jpg",)):
|
|
"""An album with verified upload evidence — what archiving requires."""
|
|
folder = lib / album
|
|
folder.mkdir(parents=True, exist_ok=True)
|
|
with sf() as session:
|
|
batch_id = str(uuid.uuid4())
|
|
session.add(
|
|
UploadBatch(
|
|
id=batch_id,
|
|
album=album,
|
|
folder=str(folder),
|
|
album_name=album,
|
|
state="succeeded",
|
|
preflight_token="v1:test",
|
|
outcome_state="verified",
|
|
created_at=NOW,
|
|
)
|
|
)
|
|
for name in names:
|
|
path = folder / name
|
|
path.write_bytes(f"{album}/{name} content".encode() * 8)
|
|
asset_id = str(uuid.uuid4())
|
|
session.add(
|
|
Asset(
|
|
id=asset_id,
|
|
original_path=str(path),
|
|
current_path=str(path),
|
|
discovered_at=NOW,
|
|
hash_version=1,
|
|
byte_size=path.stat().st_size,
|
|
current_sha256=hashing.sha256_file(path),
|
|
)
|
|
)
|
|
session.add(
|
|
UploadItem(
|
|
batch_id=batch_id,
|
|
asset_id=asset_id,
|
|
path=str(path),
|
|
sha256=hashing.sha256_file(path),
|
|
sha1="0" * 40,
|
|
state="sent",
|
|
outcome="uploaded",
|
|
)
|
|
)
|
|
session.commit()
|
|
return folder
|
|
|
|
|
|
def archive_plan(sf, config, archive, albums=None):
|
|
location = ArchiveService(sf, config=config).register("external", str(archive))
|
|
token = ArchiveService(sf, config=config).preflight(location["id"], albums)["token"]
|
|
service = ArchiveTransferService(sf, config=config)
|
|
return service, service.create(location["id"], albums, token=token)
|
|
|
|
|
|
def fake_tool(directory: Path, name: str, body: str) -> Path:
|
|
"""A real executable on a directory a test can put in front of PATH."""
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
path = directory / name
|
|
path.write_text(f"#!/bin/sh\n{body}")
|
|
path.chmod(path.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
|
|
return path
|
|
|
|
|
|
# ── control points ───────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_the_fault_barrier_does_nothing_unless_its_variable_names_the_point(monkeypatch):
|
|
monkeypatch.delenv(faults.ENV_VAR, raising=False)
|
|
for point in (faults.EXIF_WRITTEN, faults.UPLOAD_ACCEPTED, faults.JOB_ITEM_DONE, "moving"):
|
|
faults.maybe_fault(point) # would kill the process if it were armed
|
|
monkeypatch.setenv(faults.ENV_VAR, faults.EXIF_WRITTEN)
|
|
faults.maybe_fault(faults.UPLOAD_ACCEPTED) # a different point stays inert
|
|
|
|
|
|
def test_no_route_or_configuration_can_arm_a_fault():
|
|
"""The control points are reachable only through an environment variable read
|
|
inside ``photo_pipeline.faults`` — never through the API, and never through
|
|
configuration a browser or a config file could set."""
|
|
from photo_pipeline.api.app import create_app
|
|
|
|
app = create_app()
|
|
assert not [route for route in app.routes if "fault" in getattr(route, "path", "")]
|
|
assert not [field for field in Config.model_fields if "fault" in field]
|
|
|
|
package = Path(__file__).resolve().parents[2] / "photo_pipeline"
|
|
sources = {
|
|
path.relative_to(package.parent)
|
|
for path in package.rglob("*.py")
|
|
if faults.ENV_VAR in path.read_text()
|
|
}
|
|
assert sources == {Path("photo_pipeline/faults.py")}
|
|
|
|
|
|
# ── disk full ────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_a_full_disk_during_an_archive_never_removes_the_source(tmp_path, monkeypatch):
|
|
config, sf, lib = _env(tmp_path)
|
|
archive = tmp_path / "archive"
|
|
archive.mkdir()
|
|
folder = uploaded_album(sf, lib)
|
|
original = {path: path.read_bytes() for path in folder.iterdir()}
|
|
service, plan = archive_plan(sf, config, archive)
|
|
|
|
def no_space(*args, **kwargs):
|
|
raise OSError(errno.ENOSPC, "No space left on device")
|
|
|
|
# Force the cross-filesystem path (a real archive medium) and fill it up.
|
|
monkeypatch.setattr(archive_transfer, "_same_filesystem", lambda *a: False)
|
|
monkeypatch.setattr(archive_transfer.shutil, "copyfileobj", no_space)
|
|
|
|
result = service.apply(plan["id"])
|
|
|
|
assert result["archived"] == 0 and result["failed"] == 1
|
|
for path, body in original.items():
|
|
assert path.read_bytes() == body, "the source was touched despite the failure"
|
|
with sf() as session:
|
|
assert all(a.availability_state == "active" for a in session.scalars(select(Asset)))
|
|
assert [p for p in archive.rglob("*") if p.is_file() and not p.name.startswith(".")] == []
|
|
|
|
|
|
# ── read-only paths ──────────────────────────────────────────────────────────
|
|
|
|
|
|
@pytest.mark.skipif(os.geteuid() == 0, reason="root ignores directory permissions")
|
|
def test_a_read_only_library_refuses_the_rename_and_keeps_the_source(tmp_path):
|
|
config, sf, lib = _env(tmp_path)
|
|
folder = lib / "rome"
|
|
image(folder / "a.jpg")
|
|
register(sf, folder / "a.jpg")
|
|
with sf() as session:
|
|
session.add(
|
|
AlbumProposal(
|
|
id=str(uuid.uuid4()),
|
|
album="rome",
|
|
proposed_name="2019 Rome",
|
|
final_name="2019 Rome",
|
|
status="approved",
|
|
version=2,
|
|
)
|
|
)
|
|
session.commit()
|
|
plan = RenameService(sf, library_roots=(lib,)).build_plan()
|
|
|
|
mode = lib.stat().st_mode
|
|
lib.chmod(0o500) # readable, traversable, not writable
|
|
try:
|
|
result = RenameApplyService(sf, library_roots=(lib,)).apply(
|
|
plan["id"], expected_version=plan["version"]
|
|
)
|
|
finally:
|
|
lib.chmod(mode)
|
|
|
|
assert result["applied"] == 0 and result["failed"] == 1
|
|
assert (folder / "a.jpg").exists() and not (lib / "2019 Rome").exists()
|
|
with sf() as session:
|
|
asset = session.scalars(select(Asset)).one()
|
|
assert asset.current_path == str(folder / "a.jpg")
|
|
|
|
|
|
# ── database faults ──────────────────────────────────────────────────────────
|
|
|
|
|
|
@contextmanager
|
|
def exclusive_lock(database_url: str):
|
|
"""Hold SQLite's write lock from another thread, the way a second process would.
|
|
|
|
A sqlite3 connection belongs to the thread that opened it, so the holder thread
|
|
opens, locks, waits, and releases entirely on its own.
|
|
"""
|
|
path = database_url.replace("sqlite:///", "")
|
|
locked, release = threading.Event(), threading.Event()
|
|
|
|
def hold() -> None:
|
|
connection = sqlite3.connect(path, timeout=10)
|
|
connection.execute("BEGIN EXCLUSIVE")
|
|
locked.set()
|
|
release.wait(30)
|
|
connection.rollback()
|
|
connection.close()
|
|
|
|
thread = threading.Thread(target=hold)
|
|
thread.start()
|
|
assert locked.wait(10), "the holder never acquired the lock"
|
|
try:
|
|
yield release.set # callers may release early; exiting releases anyway
|
|
finally:
|
|
release.set()
|
|
thread.join(10)
|
|
|
|
|
|
def test_a_busy_database_waits_rather_than_failing(tmp_path):
|
|
"""SQLite has one writer. A short conflict must resolve through the busy
|
|
timeout instead of surfacing as an error."""
|
|
config, sf, lib = _env(tmp_path)
|
|
service = JobService(sf)
|
|
job = service.enqueue("scan", items=["a"])
|
|
|
|
with exclusive_lock(config.database_url) as release:
|
|
threading.Timer(0.3, release).start()
|
|
started = time.monotonic()
|
|
claimed = service.claim(["scan"], "w1") # blocks until the lock is gone
|
|
waited = time.monotonic() - started
|
|
|
|
assert claimed["id"] == job["id"] and claimed["state"] == "running"
|
|
assert waited >= 0.25, "the claim did not actually wait for the writer"
|
|
with sf() as session:
|
|
assert session.execute(text("PRAGMA busy_timeout")).scalar() >= 1000
|
|
|
|
|
|
def test_a_database_locked_beyond_the_timeout_is_an_error_not_a_silent_skip(tmp_path):
|
|
config, sf, lib = _env(tmp_path)
|
|
with sf() as session:
|
|
session.execute(text("SELECT 1")) # connect first: the lock comes after
|
|
with exclusive_lock(config.database_url):
|
|
session.execute(text("PRAGMA busy_timeout=50")) # do not wait five seconds
|
|
with pytest.raises(OperationalError, match="locked"):
|
|
session.execute(
|
|
text("INSERT INTO jobs (id, job_type, state) VALUES ('x','scan','queued')")
|
|
)
|
|
session.commit()
|
|
session.rollback()
|
|
|
|
# The refusal left nothing behind, and the database is still sound.
|
|
with sf() as session:
|
|
assert session.execute(text("PRAGMA integrity_check")).scalar() == "ok"
|
|
assert session.execute(text("SELECT count(*) FROM jobs")).scalar() == 0
|
|
|
|
|
|
def test_a_corrupt_database_fails_loudly_instead_of_answering_wrongly(tmp_path):
|
|
config, sf, lib = _env(tmp_path)
|
|
for index in range(50): # enough rows to fill several pages
|
|
JobService(sf).enqueue("scan", items=[f"item-{index}-{n}" for n in range(20)])
|
|
with sf() as session:
|
|
session.execute(text("PRAGMA wal_checkpoint(TRUNCATE)")) # WAL into the file
|
|
session.commit()
|
|
source = Path(config.database_url.replace("sqlite:///", ""))
|
|
|
|
corrupt = tmp_path / "corrupt.db"
|
|
body = bytearray(source.read_bytes())
|
|
body[4096 : 4096 + 2048] = b"\xde\xad\xbe\xef" * 512 # shred pages, keep the header
|
|
corrupt.write_bytes(bytes(body))
|
|
|
|
engine = create_db_engine(f"sqlite:///{corrupt}")
|
|
factory = create_session_factory(engine)
|
|
try:
|
|
with factory() as session:
|
|
assert session.execute(text("PRAGMA integrity_check")).scalar() != "ok"
|
|
# Reading the shredded pages must raise, never return half a table.
|
|
with pytest.raises(DatabaseError):
|
|
session.execute(text("SELECT * FROM job_items")).all()
|
|
session.execute(text("SELECT * FROM job_events")).all()
|
|
session.execute(text("REINDEX")).all()
|
|
finally:
|
|
engine.dispose()
|
|
|
|
|
|
# ── external services ────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_an_unreachable_immich_blocks_upload_instead_of_starting_one(tmp_path):
|
|
# Port 9 (discard) refuses connections deterministically.
|
|
config, sf, lib = _env(
|
|
tmp_path,
|
|
PHOTO_PIPELINE_IMMICH_SERVER_URL="http://127.0.0.1:9",
|
|
PHOTO_PIPELINE_IMMICH_API_KEY="sentinel",
|
|
)
|
|
report = UploadService(sf, config=config).preflight()
|
|
codes = {blocker["code"] for blocker in report["blockers"]}
|
|
assert "server_unreachable" in codes
|
|
assert report["state"] != "ready"
|
|
|
|
|
|
def test_a_missing_uploader_blocks_upload_with_the_binary_named(tmp_path):
|
|
config, sf, lib = _env(
|
|
tmp_path,
|
|
PHOTO_PIPELINE_IMMICH_SERVER_URL="http://127.0.0.1:9",
|
|
PHOTO_PIPELINE_IMMICH_API_KEY="sentinel",
|
|
PHOTO_PIPELINE_IMMICH_GO_BINARY=str(tmp_path / "no-such-immich-go"),
|
|
)
|
|
report = UploadService(sf, config=config).preflight()
|
|
assert "immich_go_missing" in {blocker["code"] for blocker in report["blockers"]}
|
|
|
|
|
|
def test_a_malformed_provider_answer_is_a_per_asset_error(tmp_path):
|
|
config, sf, lib = _env(tmp_path)
|
|
path = lib / "a.jpg"
|
|
image(path)
|
|
asset_id = register(sf, path)
|
|
with sf() as session:
|
|
session.add(
|
|
SafetyReview(id=str(uuid.uuid4()), asset_id=asset_id, decision="sfw", created_at=NOW)
|
|
)
|
|
session.commit()
|
|
|
|
class MalformedProvider:
|
|
def analyze(self, path, *, album_hint):
|
|
raise ValueError("Expecting value: line 1 column 1 (char 0)")
|
|
|
|
result = AnalysisService(sf, provider=MalformedProvider(), library_roots=(lib,)).run([asset_id])
|
|
|
|
assert result == {"analyzed": 0, "skipped": 0, "errors": 1}
|
|
with sf() as session:
|
|
row = session.get(AnalysisResult, asset_id)
|
|
assert row.status == "error" and "Expecting value" in row.error_message
|
|
assert session.get(ExifProjection, (asset_id, "analysis")) is None
|
|
|
|
|
|
def test_gpu_exhaustion_fails_the_item_without_inventing_a_score(tmp_path):
|
|
config, sf, lib = _env(tmp_path)
|
|
path = lib / "a.jpg"
|
|
image(path)
|
|
asset_id = register(sf, path)
|
|
|
|
class ExhaustedModel:
|
|
def score(self, paths):
|
|
raise RuntimeError("MPS backend out of memory (MPS allocated: 9.00 GB)")
|
|
|
|
service = SafetyService(sf, model=ExhaustedModel())
|
|
jobs = JobService(sf)
|
|
job = jobs.enqueue("safety_score", items=[asset_id])
|
|
Worker(sf, {"safety_score": lambda item, ctx: service.score_assets([item])}, "w1").run_once()
|
|
|
|
progress = jobs.progress(job["id"])
|
|
assert progress["by_state"] == {ItemState.FAILED: 1}
|
|
with sf() as session:
|
|
assert session.scalars(select(SafetyReview)).all() == [], "no score was invented"
|
|
|
|
|
|
# ── external tools ───────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_a_hanging_exiftool_times_out_and_verifies_nothing(tmp_path, monkeypatch):
|
|
config, sf, lib = _env(tmp_path)
|
|
path = lib / "a.jpg"
|
|
image(path)
|
|
asset_id = register(sf, path)
|
|
before = hashing.sha256_file(path)
|
|
|
|
fake_tool(tmp_path / "bin", "exiftool", "sleep 30\n")
|
|
monkeypatch.setenv("PATH", f"{tmp_path / 'bin'}:{os.environ['PATH']}")
|
|
monkeypatch.setenv("PHOTO_PIPELINE_EXIFTOOL_TIMEOUT", "1")
|
|
|
|
review = SafetyService(sf).decide(asset_id, "nsfw")
|
|
|
|
# The decision is durable; the metadata claim is not made.
|
|
assert review["decision"] == "nsfw" and review["exif_verified"] is False
|
|
assert exif_checkpoint.state_for(sf, asset_id, "safety") == exif_checkpoint.FAILED
|
|
assert hashing.sha256_file(path) == before
|
|
|
|
|
|
def test_a_missing_exiftool_is_a_failed_checkpoint_not_a_verified_one(tmp_path, monkeypatch):
|
|
config, sf, lib = _env(tmp_path)
|
|
path = lib / "a.jpg"
|
|
image(path)
|
|
asset_id = register(sf, path)
|
|
|
|
empty_bin = tmp_path / "empty-bin"
|
|
empty_bin.mkdir()
|
|
monkeypatch.setenv("PATH", str(empty_bin)) # nothing on PATH at all
|
|
|
|
review = SafetyService(sf).decide(asset_id, "sfw")
|
|
|
|
assert review["exif_verified"] is False
|
|
assert exif_checkpoint.state_for(sf, asset_id, "safety") == exif_checkpoint.FAILED
|
|
with sf() as session:
|
|
# Upload eligibility depends on a verified checkpoint, so it stays blocked.
|
|
assert session.scalars(select(SafetyReview)).all()[-1].exif_verified_at is None
|
|
|
|
|
|
def test_a_file_edited_during_the_move_is_left_for_a_human(tmp_path, monkeypatch):
|
|
"""The user saves over a photo in the instant between the move and its
|
|
verification. The move already happened and the database already followed it,
|
|
so the operation cannot simply be "failed": it becomes ``rollback_required``
|
|
and blocks further mutation until someone decides (US07-04)."""
|
|
config, sf, lib = _env(tmp_path)
|
|
folder = lib / "rome"
|
|
image(folder / "a.jpg")
|
|
register(sf, folder / "a.jpg")
|
|
with sf() as session:
|
|
session.add(
|
|
AlbumProposal(
|
|
id=str(uuid.uuid4()),
|
|
album="rome",
|
|
proposed_name="2019 Rome",
|
|
final_name="2019 Rome",
|
|
status="approved",
|
|
version=2,
|
|
)
|
|
)
|
|
session.commit()
|
|
plan = RenameService(sf, library_roots=(lib,)).build_plan()
|
|
|
|
real_rename = rename_apply.os.rename
|
|
|
|
def rename_then_edit(source, destination):
|
|
real_rename(source, destination)
|
|
for path in Path(destination).glob("*.jpg"):
|
|
path.write_bytes(b"the user saved over this file")
|
|
|
|
monkeypatch.setattr(rename_apply.os, "rename", rename_then_edit)
|
|
|
|
result = RenameApplyService(sf, library_roots=(lib,)).apply(
|
|
plan["id"], expected_version=plan["version"]
|
|
)
|
|
|
|
assert result["applied"] == 0 and result["failed"] == 1
|
|
journal = RenameJournal(sf)
|
|
operation = journal.incomplete()[0]
|
|
assert operation["journal_state"] == JournalState.ROLLBACK_REQUIRED
|
|
assert operation["error_code"] == "verify_bytes"
|
|
assert journal.blocks_mutation() is True, "the unresolved rename blocks the library"
|
|
|
|
# Recovery offers the rollback the evidence supports, and the rollback itself
|
|
# refuses the edited bytes rather than putting the user's newer file back as if
|
|
# it were the old one.
|
|
service = RenameApplyService(sf, library_roots=(lib,))
|
|
with pytest.raises(ApplyError, match="manual recovery"):
|
|
service.rollback_operation(operation["id"])
|
|
# Nothing was lost: the edited file is at its new home, not deleted.
|
|
assert (lib / "2019 Rome" / "a.jpg").read_bytes() == b"the user saved over this file"
|