"""Process test: kill a worker mid-item, then prove the job recovers and a fresh worker finishes it. The killed worker's lease expires and its ownership is fenced.""" import subprocess import sys import time from pathlib import Path import pytest from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations from photo_pipeline.jobs.worker import Worker from photo_pipeline.services.jobs import ItemState, JobService, JobState REPO = Path(__file__).resolve().parents[2] # Worker-recovery is also part of the Phase B acceptance command (US02-07: the # "resume" journey); it stays mapped to US02-03 for traceability. pytestmark = pytest.mark.phase_b # A worker that claims the job, marks its item running, signals it started, then # blocks — so the test can kill it mid-item. WORKER_SCRIPT = """ import sys, time from pathlib import Path sys.path.insert(0, {repo!r}) from photo_pipeline.db import create_db_engine, create_session_factory from photo_pipeline.jobs.worker import Worker db_url, started = sys.argv[1], Path(sys.argv[2]) def blocking_handler(item_key, ctx): started.write_text("started") time.sleep(30) sf = create_session_factory(create_db_engine(db_url)) Worker(sf, {{"slow": blocking_handler}}, "killable", lease_seconds=1).run_once() """ @pytest.fixture def db_url(tmp_path): url = f"sqlite:///{tmp_path / 'kill.db'}" run_migrations(url) return url def test_killed_worker_job_recovers_and_completes(tmp_path, db_url): engine = create_db_engine(db_url) service = JobService(create_session_factory(engine)) job = service.enqueue("slow", items=["only"]) script = tmp_path / "worker_script.py" script.write_text(WORKER_SCRIPT.format(repo=str(REPO))) started = tmp_path / "started.flag" proc = subprocess.Popen( [sys.executable, str(script), db_url, str(started)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) try: deadline = time.monotonic() + 20 while time.monotonic() < deadline and not started.exists(): if proc.poll() is not None: _, err = proc.communicate() pytest.fail(f"worker exited early: {err.decode(errors='replace')}") time.sleep(0.1) assert started.exists(), "worker never began the item" assert service.get(job["id"])["state"] == JobState.RUNNING finally: proc.kill() proc.wait(timeout=10) # Let the 1s lease expire, then a fresh worker recovers and finishes the job. time.sleep(1.3) completed = [] fresh = Worker( create_session_factory(engine), {"slow": lambda item, ctx: completed.append(item)}, "recovery-worker", ) fresh.run_once() assert service.get(job["id"])["state"] == JobState.SUCCEEDED assert completed == ["only"] # the interrupted item was reprocessed assert service.progress(job["id"])["by_state"] == {ItemState.SUCCEEDED: 1} engine.dispose()