US02-03: Execute Jobs with Leases, Locks, and Recovery (#56)

tests/e2e/test_worker_kill.py

@@ -0,0 +1,84 @@
+"""Process test: kill a worker mid-item, then prove the job recovers and a fresh
+worker finishes it. The killed worker's lease expires and its ownership is fenced."""
+
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+import pytest
+
+from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
+from photo_pipeline.jobs.worker import Worker
+from photo_pipeline.services.jobs import ItemState, JobService, JobState
+
+REPO = Path(__file__).resolve().parents[2]
+
+# A worker that claims the job, marks its item running, signals it started, then
+# blocks — so the test can kill it mid-item.
+WORKER_SCRIPT = """
+import sys, time
+from pathlib import Path
+sys.path.insert(0, {repo!r})
+from photo_pipeline.db import create_db_engine, create_session_factory
+from photo_pipeline.jobs.worker import Worker
+
+db_url, started = sys.argv[1], Path(sys.argv[2])
+
+def blocking_handler(item_key, ctx):
+    started.write_text("started")
+    time.sleep(30)
+
+sf = create_session_factory(create_db_engine(db_url))
+Worker(sf, {{"slow": blocking_handler}}, "killable", lease_seconds=1).run_once()
+"""
+
+
+@pytest.fixture
+def db_url(tmp_path):
+    url = f"sqlite:///{tmp_path / 'kill.db'}"
+    run_migrations(url)
+    return url
+
+
+def test_killed_worker_job_recovers_and_completes(tmp_path, db_url):
+    engine = create_db_engine(db_url)
+    service = JobService(create_session_factory(engine))
+    job = service.enqueue("slow", items=["only"])
+
+    script = tmp_path / "worker_script.py"
+    script.write_text(WORKER_SCRIPT.format(repo=str(REPO)))
+    started = tmp_path / "started.flag"
+
+    proc = subprocess.Popen(
+        [sys.executable, str(script), db_url, str(started)],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    try:
+        deadline = time.monotonic() + 20
+        while time.monotonic() < deadline and not started.exists():
+            if proc.poll() is not None:
+                _, err = proc.communicate()
+                pytest.fail(f"worker exited early: {err.decode(errors='replace')}")
+            time.sleep(0.1)
+        assert started.exists(), "worker never began the item"
+        assert service.get(job["id"])["state"] == JobState.RUNNING
+    finally:
+        proc.kill()
+        proc.wait(timeout=10)
+
+    # Let the 1s lease expire, then a fresh worker recovers and finishes the job.
+    time.sleep(1.3)
+    completed = []
+    fresh = Worker(
+        create_session_factory(engine),
+        {"slow": lambda item, ctx: completed.append(item)},
+        "recovery-worker",
+    )
+    fresh.run_once()
+
+    assert service.get(job["id"])["state"] == JobState.SUCCEEDED
+    assert completed == ["only"]  # the interrupted item was reprocessed
+    assert service.progress(job["id"])["by_state"] == {ItemState.SUCCEEDED: 1}
+    engine.dispose()