89 lines
2.9 KiB
Python
89 lines
2.9 KiB
Python
"""Process test: kill a worker mid-item, then prove the job recovers and a fresh
|
|
worker finishes it. The killed worker's lease expires and its ownership is fenced."""
|
|
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
|
|
from photo_pipeline.jobs.worker import Worker
|
|
from photo_pipeline.services.jobs import ItemState, JobService, JobState
|
|
|
|
REPO = Path(__file__).resolve().parents[2]
|
|
|
|
# Worker-recovery is also part of the Phase B acceptance command (US02-07: the
|
|
# "resume" journey); it stays mapped to US02-03 for traceability.
|
|
pytestmark = pytest.mark.phase_b
|
|
|
|
# A worker that claims the job, marks its item running, signals it started, then
|
|
# blocks — so the test can kill it mid-item.
|
|
WORKER_SCRIPT = """
|
|
import sys, time
|
|
from pathlib import Path
|
|
sys.path.insert(0, {repo!r})
|
|
from photo_pipeline.db import create_db_engine, create_session_factory
|
|
from photo_pipeline.jobs.worker import Worker
|
|
|
|
db_url, started = sys.argv[1], Path(sys.argv[2])
|
|
|
|
def blocking_handler(item_key, ctx):
|
|
started.write_text("started")
|
|
time.sleep(30)
|
|
|
|
sf = create_session_factory(create_db_engine(db_url))
|
|
Worker(sf, {{"slow": blocking_handler}}, "killable", lease_seconds=1).run_once()
|
|
"""
|
|
|
|
|
|
@pytest.fixture
|
|
def db_url(tmp_path):
|
|
url = f"sqlite:///{tmp_path / 'kill.db'}"
|
|
run_migrations(url)
|
|
return url
|
|
|
|
|
|
def test_killed_worker_job_recovers_and_completes(tmp_path, db_url):
|
|
engine = create_db_engine(db_url)
|
|
service = JobService(create_session_factory(engine))
|
|
job = service.enqueue("slow", items=["only"])
|
|
|
|
script = tmp_path / "worker_script.py"
|
|
script.write_text(WORKER_SCRIPT.format(repo=str(REPO)))
|
|
started = tmp_path / "started.flag"
|
|
|
|
proc = subprocess.Popen(
|
|
[sys.executable, str(script), db_url, str(started)],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
try:
|
|
deadline = time.monotonic() + 20
|
|
while time.monotonic() < deadline and not started.exists():
|
|
if proc.poll() is not None:
|
|
_, err = proc.communicate()
|
|
pytest.fail(f"worker exited early: {err.decode(errors='replace')}")
|
|
time.sleep(0.1)
|
|
assert started.exists(), "worker never began the item"
|
|
assert service.get(job["id"])["state"] == JobState.RUNNING
|
|
finally:
|
|
proc.kill()
|
|
proc.wait(timeout=10)
|
|
|
|
# Let the 1s lease expire, then a fresh worker recovers and finishes the job.
|
|
time.sleep(1.3)
|
|
completed = []
|
|
fresh = Worker(
|
|
create_session_factory(engine),
|
|
{"slow": lambda item, ctx: completed.append(item)},
|
|
"recovery-worker",
|
|
)
|
|
fresh.run_once()
|
|
|
|
assert service.get(job["id"])["state"] == JobState.SUCCEEDED
|
|
assert completed == ["only"] # the interrupted item was reprocessed
|
|
assert service.progress(job["id"])["by_state"] == {ItemState.SUCCEEDED: 1}
|
|
engine.dispose()
|