US02-03: Execute Jobs with Leases, Locks, and Recovery (#56)

photo_pipeline/services/jobs.py

@@ -66,8 +66,9 @@ ALLOWED_TRANSITIONS = {
 }
 
 ITEM_TRANSITIONS = {
+    # running -> queued is an interrupted item being made resumable again.
     ItemState.QUEUED: {ItemState.RUNNING, ItemState.CANCELLED},
-    ItemState.RUNNING: {ItemState.SUCCEEDED, ItemState.FAILED, ItemState.CANCELLED},
+    ItemState.RUNNING: {ItemState.SUCCEEDED, ItemState.FAILED, ItemState.CANCELLED, ItemState.QUEUED},
     ItemState.FAILED: {ItemState.QUEUED, ItemState.RUNNING},
     ItemState.SUCCEEDED: set(),
     ItemState.CANCELLED: set(),
@@ -219,6 +220,7 @@ class JobService:
         to_state: str,
         *,
         worker_id: str | None = None,
+        fencing_token: int | None = None,
         error: tuple[str, str] | None = None,
     ) -> dict:
         now = _now()
@@ -226,6 +228,10 @@ class JobService:
             job = session.get(Job, job_id)
             if job is None:
                 raise JobError(f"unknown job {job_id}")
+            if fencing_token is not None and job.fencing_token != fencing_token:
+                raise JobConflict(
+                    f"stale fencing token {fencing_token} (current {job.fencing_token})"
+                )
             if not can_transition(job.state, to_state):
                 raise InvalidTransition(f"{job.state} -> {to_state}")
             if worker_id is not None and job.lease_owner not in (None, worker_id):
@@ -258,10 +264,24 @@ class JobService:
 
     # ── items / progress ───────────────────────────────────────────────────────
     def set_item(
-        self, job_id: str, item_key: str, state: str, *, error: tuple[str, str] | None = None
+        self,
+        job_id: str,
+        item_key: str,
+        state: str,
+        *,
+        error: tuple[str, str] | None = None,
+        fencing_token: int | None = None,
     ) -> None:
         now = _now()
         with self._session_factory() as session:
+            if fencing_token is not None:
+                job = session.get(Job, job_id)
+                if job is None:
+                    raise JobError(f"unknown job {job_id}")
+                if job.fencing_token != fencing_token:
+                    raise JobConflict(
+                        f"stale fencing token {fencing_token} (current {job.fencing_token})"
+                    )
             item = session.get(JobItem, {"job_id": job_id, "item_key": item_key})
             if item is None:
                 raise JobError(f"unknown item {job_id}/{item_key}")