US02-03: Execute Jobs with Leases, Locks, and Recovery (#56)

photo_pipeline/__main__.py

@@ -14,6 +14,8 @@ def main(argv: Sequence[str] | None = None) -> int:
     commands = parser.add_subparsers(dest="command", required=True)
     commands.add_parser("serve", help="Run the API server")
     commands.add_parser("migrate", help="Upgrade the database to the latest revision")
+    worker_cmd = commands.add_parser("worker", help="Run a durable-job worker")
+    worker_cmd.add_argument("--id", default="worker-1", help="Worker id (lease owner)")
     args = parser.parse_args(argv)
 
     config = Config.from_env()
@@ -23,6 +25,15 @@ def main(argv: Sequence[str] | None = None) -> int:
         run_migrations(config.database_url)
         return 0
 
+    if args.command == "worker":
+        from photo_pipeline.db import create_db_engine, create_session_factory
+        from photo_pipeline.jobs.worker import Worker
+
+        run_migrations(config.database_url)
+        engine = create_db_engine(config.database_url)
+        Worker(create_session_factory(engine), worker_id=args.id).run_forever()
+        return 0
+
     import uvicorn
 
     from photo_pipeline.api.app import create_app

photo_pipeline/jobs/__init__.py

@@ -0,0 +1 @@
+"""Durable job execution: worker loop, handler dispatch, and lock ordering."""

photo_pipeline/jobs/handlers.py

@@ -0,0 +1,54 @@
+"""Job-type handler dispatch.
+
+A handler processes one item of a job: ``handler(item_key, ctx)`` and either
+returns (success), raises ``Cancelled`` (cooperative stop, resumable), or raises
+any other exception (item failure). Handlers must be idempotent — at-least-once
+delivery means the same item can run again after an interrupted attempt.
+
+Domain handlers (safety scoring, analysis, …) register here in later stories; the
+registry ships empty so the worker engine can be built and tested independently.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Callable
+
+if TYPE_CHECKING:
+    from photo_pipeline.services.jobs import JobService
+
+Handler = Callable[[str, "JobContext"], None]
+
+
+class Cancelled(Exception):
+    """Raised by a handler that observed cancellation and stopped cleanly."""
+
+
+@dataclass
+class JobContext:
+    """What a handler needs from the coordinator: cancellation checks and heartbeats."""
+
+    job_id: str
+    worker_id: str
+    fencing_token: int
+    service: "JobService"
+
+    def cancelled(self) -> bool:
+        from photo_pipeline.services.jobs import JobState
+
+        snapshot = self.service.get(self.job_id)
+        return snapshot is None or snapshot["state"] in (
+            JobState.CANCELLING,
+            JobState.CANCELLED,
+        )
+
+    def heartbeat(self, *, lease_seconds: int = 60) -> bool:
+        return self.service.heartbeat(self.job_id, self.worker_id, lease_seconds=lease_seconds)
+
+
+# Populated by domain stories via register(); empty for now.
+REGISTRY: dict[str, Handler] = {}
+
+
+def register(job_type: str, handler: Handler) -> None:
+    REGISTRY[job_type] = handler

photo_pipeline/jobs/locks.py

@@ -0,0 +1,44 @@
+"""Lock hierarchy and acquisition ordering.
+
+Locks are acquired broad → narrow to prevent deadlocks: never take a broader lock
+while holding a narrower one (concept §16). The coordinator already enforces
+one active job per lock key; this guards multi-lock operations.
+
+    library lease → stage/job lease → album/folder lease → asset lease
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+# Lower rank = broader scope. Mutating lanes map onto these tiers.
+LOCK_RANK = {
+    "library": 0,
+    "library_write": 0,
+    "rename": 1,
+    "upload": 1,
+    "archive": 1,
+    "album": 2,
+    "asset": 3,
+    "exif": 3,
+}
+
+
+class LockOrderError(RuntimeError):
+    pass
+
+
+def rank(lock: str) -> int:
+    if lock not in LOCK_RANK:
+        raise LockOrderError(f"unknown lock {lock!r}")
+    return LOCK_RANK[lock]
+
+
+def validate_acquisition(held: Iterable[str], acquiring: str) -> None:
+    """Reject acquiring a broader lock than one already held (deadlock risk)."""
+    new_rank = rank(acquiring)
+    for lock in held:
+        if rank(lock) > new_rank:
+            raise LockOrderError(
+                f"cannot acquire broader lock {acquiring!r} while holding narrower {lock!r}"
+            )

photo_pipeline/jobs/worker.py

@@ -0,0 +1,138 @@
+"""Durable worker: claims jobs and runs their items safely.
+
+Each worker serves a set of job types (its lanes), claims one job at a time (a
+bounded single lane), and processes items sequentially with heartbeats. It holds
+the fencing token from its claim and stamps every state write with it, so a worker
+that was superseded after a lease expiry cannot commit stale results. Cancellation
+is cooperative (checked between items and offered to handlers), leaving items
+resumable. On resume, items left ``running`` by a dead worker are reset to
+``queued`` and re-run (handlers are idempotent).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+
+from sqlalchemy import select
+from sqlalchemy.orm import sessionmaker
+
+from photo_pipeline.jobs.handlers import REGISTRY, Cancelled, Handler, JobContext
+from photo_pipeline.models import JobItem
+from photo_pipeline.services.jobs import ItemState, JobConflict, JobService, JobState
+
+
+class Worker:
+    def __init__(
+        self,
+        session_factory: sessionmaker,
+        handlers: Mapping[str, Handler] | None = None,
+        worker_id: str = "worker",
+        *,
+        job_types: Sequence[str] | None = None,
+        lease_seconds: int = 60,
+    ) -> None:
+        self._session_factory = session_factory
+        self.service = JobService(session_factory)
+        self.handlers: Mapping[str, Handler] = handlers if handlers is not None else REGISTRY
+        self.worker_id = worker_id
+        self.job_types = list(job_types if job_types is not None else self.handlers.keys())
+        self.lease_seconds = lease_seconds
+
+    def run_once(self) -> str | None:
+        """Recover stragglers, then claim and fully process one job. Returns its id."""
+        self.service.recover_stale()
+        claimed = self.service.claim(
+            self.job_types, self.worker_id, lease_seconds=self.lease_seconds
+        )
+        if claimed is None:
+            return None
+        self._process(claimed["id"], claimed["job_type"], claimed["fencing_token"])
+        return claimed["id"]
+
+    def _process(self, job_id: str, job_type: str, token: int) -> None:
+        handler = self.handlers[job_type]
+        ctx = JobContext(job_id, self.worker_id, token, self.service)
+        self._reset_interrupted_items(job_id, token)
+
+        cancelled = False
+        any_failed = False
+        for item_key in self._queued_items(job_id):
+            if ctx.cancelled():
+                cancelled = True
+                break
+            try:
+                self.service.set_item(job_id, item_key, ItemState.RUNNING, fencing_token=token)
+                handler(item_key, ctx)
+            except Cancelled:
+                # Cooperative stop: leave the item resumable.
+                self.service.set_item(job_id, item_key, ItemState.QUEUED, fencing_token=token)
+                cancelled = True
+                break
+            except JobConflict:
+                # Superseded mid-item; stop and let the new owner finish.
+                return
+            except Exception as error:  # handler failure for this item
+                any_failed = True
+                self.service.set_item(
+                    job_id,
+                    item_key,
+                    ItemState.FAILED,
+                    error=("handler_error", str(error)[:200]),
+                    fencing_token=token,
+                )
+            else:
+                self.service.set_item(job_id, item_key, ItemState.SUCCEEDED, fencing_token=token)
+            self.service.heartbeat(job_id, self.worker_id, lease_seconds=self.lease_seconds)
+
+        self._finalize(job_id, token, cancelled=cancelled, any_failed=any_failed)
+
+    def _finalize(self, job_id: str, token: int, *, cancelled: bool, any_failed: bool) -> None:
+        snapshot = self.service.get(job_id)
+        if snapshot is None:
+            return
+        try:
+            if cancelled or snapshot["state"] == JobState.CANCELLING:
+                self.service.transition(
+                    job_id, JobState.CANCELLED, worker_id=self.worker_id, fencing_token=token
+                )
+            elif any_failed:
+                self.service.transition(
+                    job_id,
+                    JobState.FAILED,
+                    worker_id=self.worker_id,
+                    fencing_token=token,
+                    error=("items_failed", "one or more items failed"),
+                )
+            else:
+                self.service.transition(
+                    job_id, JobState.SUCCEEDED, worker_id=self.worker_id, fencing_token=token
+                )
+        except JobConflict:
+            # A newer worker owns the job now; do not overwrite its outcome.
+            return
+
+    def _reset_interrupted_items(self, job_id: str, token: int) -> None:
+        for item_key in self._items_in_state(job_id, ItemState.RUNNING):
+            self.service.set_item(job_id, item_key, ItemState.QUEUED, fencing_token=token)
+
+    def _queued_items(self, job_id: str) -> list[str]:
+        return self._items_in_state(job_id, ItemState.QUEUED)
+
+    def _items_in_state(self, job_id: str, state: str) -> list[str]:
+        with self._session_factory() as session:
+            return list(
+                session.execute(
+                    select(JobItem.item_key)
+                    .where(JobItem.job_id == job_id, JobItem.state == state)
+                    .order_by(JobItem.item_key)
+                ).scalars()
+            )
+
+    def run_forever(self, *, idle_sleep: float = 1.0, iterations: int | None = None) -> None:
+        import time
+
+        count = 0
+        while iterations is None or count < iterations:
+            if self.run_once() is None:
+                time.sleep(idle_sleep)
+            count += 1

photo_pipeline/services/jobs.py

@@ -66,8 +66,9 @@ ALLOWED_TRANSITIONS = {
 }
 
 ITEM_TRANSITIONS = {
+    # running -> queued is an interrupted item being made resumable again.
     ItemState.QUEUED: {ItemState.RUNNING, ItemState.CANCELLED},
-    ItemState.RUNNING: {ItemState.SUCCEEDED, ItemState.FAILED, ItemState.CANCELLED},
+    ItemState.RUNNING: {ItemState.SUCCEEDED, ItemState.FAILED, ItemState.CANCELLED, ItemState.QUEUED},
     ItemState.FAILED: {ItemState.QUEUED, ItemState.RUNNING},
     ItemState.SUCCEEDED: set(),
     ItemState.CANCELLED: set(),
@@ -219,6 +220,7 @@ class JobService:
         to_state: str,
         *,
         worker_id: str | None = None,
+        fencing_token: int | None = None,
         error: tuple[str, str] | None = None,
     ) -> dict:
         now = _now()
@@ -226,6 +228,10 @@ class JobService:
             job = session.get(Job, job_id)
             if job is None:
                 raise JobError(f"unknown job {job_id}")
+            if fencing_token is not None and job.fencing_token != fencing_token:
+                raise JobConflict(
+                    f"stale fencing token {fencing_token} (current {job.fencing_token})"
+                )
             if not can_transition(job.state, to_state):
                 raise InvalidTransition(f"{job.state} -> {to_state}")
             if worker_id is not None and job.lease_owner not in (None, worker_id):
@@ -258,10 +264,24 @@ class JobService:
 
     # ── items / progress ───────────────────────────────────────────────────────
     def set_item(
-        self, job_id: str, item_key: str, state: str, *, error: tuple[str, str] | None = None
+        self,
+        job_id: str,
+        item_key: str,
+        state: str,
+        *,
+        error: tuple[str, str] | None = None,
+        fencing_token: int | None = None,
     ) -> None:
         now = _now()
         with self._session_factory() as session:
+            if fencing_token is not None:
+                job = session.get(Job, job_id)
+                if job is None:
+                    raise JobError(f"unknown job {job_id}")
+                if job.fencing_token != fencing_token:
+                    raise JobConflict(
+                        f"stale fencing token {fencing_token} (current {job.fencing_token})"
+                    )
             item = session.get(JobItem, {"job_id": job_id, "item_key": item_key})
             if item is None:
                 raise JobError(f"unknown item {job_id}/{item_key}")