"""UploadBatchService — one approved album at a time through immich-go (US05-02). Preflight (US05-01) proves a scope is safe and issues a token; this service turns that approval into a durable batch and runs it. Upload is the one stage the app cannot undo, so the discipline is: - **the approval is re-proved, not remembered.** Before every attempt the batch's preflight token is recomputed from the current library state. Bytes edited after approval, a withdrawn safety decision, or a server that stopped answering all produce a different token and the attempt is refused, never run "optimistically". - **one lane.** A batch can only start while no other batch is running; the album scope of a batch never widens after creation (concept §16 uploader lane). - **cancellation is cooperative and durable.** ``cancel`` writes ``cancelling``; the running attempt observes it through the database, stops the uploader, and records ``cancelled``. A cancelled batch is re-runnable from its own boundary. - **an interrupted attempt is uncertain, not failed.** Immich may have accepted files the app never saw a report for, so ``recover`` marks a batch whose worker vanished ``unknown_requires_verification`` (concept §15) instead of retrying it blindly. :func:`~photo_pipeline.services.upload_verification.retry_blockers` (US05-04) is what decides whether an attempt may start at all. Per-asset upload *results* are not interpreted here: after the attempt ends the report is handed to :class:`~photo_pipeline.services.upload_reports. UploadReportService` (US05-03), which classifies every file. The batch ``state`` stays an honest description of what the *process* did; ``outcome_state`` says whether the report's evidence is complete enough to trust. """ from __future__ import annotations import json import uuid from datetime import datetime, timezone from pathlib import Path from sqlalchemy import select, update from sqlalchemy.orm import sessionmaker from photo_pipeline.config import Config from photo_pipeline.faults import UPLOAD_ACCEPTED, maybe_fault from photo_pipeline.integrations import immich_go from photo_pipeline.models import UploadBatch, UploadItem from photo_pipeline.services.hashing import sha1_file from photo_pipeline.services.upload_reports import UploadReportService from photo_pipeline.services.upload_verification import retry_blockers from photo_pipeline.services.uploads import UploadService class BatchState: PLANNED = "planned" RUNNING = "running" CANCELLING = "cancelling" CANCELLED = "cancelled" SUCCEEDED = "succeeded" FAILED = "failed" # The attempt died without a parsed outcome: the server may hold the files. UNKNOWN = "unknown_requires_verification" # States that occupy the single uploader lane. LANE_STATES = frozenset({BatchState.RUNNING, BatchState.CANCELLING}) # States a batch may be (re)started from. ``unknown_requires_verification`` is not # among them: an uncertain upload must be verified (US05-04), never blindly retried. RUNNABLE_STATES = frozenset({BatchState.PLANNED, BatchState.FAILED, BatchState.CANCELLED}) # States whose batch is still the live one for its album. OPEN_STATES = frozenset({BatchState.PLANNED, BatchState.RUNNING, BatchState.CANCELLING}) class ItemState: PENDING = "pending" # ``sent`` means the batch process exited cleanly, not that Immich confirmed the # asset — the per-item outcome comes from the report in US05-03/US05-04. SENT = "sent" FAILED = "failed" class BatchError(RuntimeError): """The request cannot be carried out (unknown batch, wrong state, blocked).""" def __init__(self, code: str, message: str) -> None: super().__init__(message) self.code = code class BatchConflict(BatchError): """The lane is busy or the approval is stale — retry after resolving it.""" def _now() -> datetime: return datetime.now(timezone.utc) class UploadBatchService: def __init__(self, session_factory: sessionmaker, *, config: Config) -> None: self._session_factory = session_factory self._config = config self._preflight = UploadService(session_factory, config=config) # ── creation ────────────────────────────────────────────────────────────── def create( self, albums: list[str] | None = None, *, token: str, allow_partial: bool = False ) -> list[dict]: """Turn a *ready* preflight into one durable batch per album. ``token`` must be the token of the current preflight for the same scope and policy; anything else means the browser is acting on a stale preview. """ report = self._preflight.preflight(albums, allow_partial=allow_partial) if token != report["token"]: raise BatchConflict( "stale_preflight", "the preflight token does not describe the current state" ) if report["state"] != "ready": raise BatchError("not_ready", "the scope has unresolved upload blockers") created: list[dict] = [] for album in report["albums"]: existing = self._open_batch_for(album["album"]) if existing is not None: created.append(existing) # idempotent: one open batch per album continue created.append(self._create_one(album, token=token, allow_partial=allow_partial)) return created def _create_one(self, album: dict, *, token: str, allow_partial: bool) -> dict: eligible = [asset for asset in album["assets"] if not asset["blockers"]] batch_id = str(uuid.uuid4()) command = immich_go.preview_command( binary=self._config.immich_go_binary, server_url=self._config.immich_server_url, album_name=album["album_name"], folder=album["folder"], ) with self._session_factory() as session: session.add( UploadBatch( id=batch_id, album=album["album"], folder=album["folder"], album_name=album["album_name"], state=BatchState.PLANNED, preflight_token=token, allow_partial=allow_partial, command=json.dumps(command), uploader_version=immich_go.version(self._config.immich_go_binary), asset_count=len(eligible), ) ) for asset in eligible: session.add( UploadItem( batch_id=batch_id, asset_id=asset["asset_id"], path=asset["current_path"], sha256=asset["current_sha256"], sha1=sha1_file(asset["current_path"]), state=ItemState.PENDING, ) ) session.commit() return self.get(batch_id) # ── running ─────────────────────────────────────────────────────────────── def run(self, batch_id: str, *, worker_id: str = "uploader", cancelled=None) -> dict: """Run one batch to completion. Blocks for the duration of the upload.""" batch = self._require(batch_id) # Retry policy (US05-04): a safe failure may run again, an uncertain outcome # or bytes edited after upload may not. blocked = retry_blockers(batch) if blocked: raise BatchError(blocked[0]["code"], blocked[0]["message"]) busy = [row for row in self.list() if row["id"] != batch_id and row["state"] in LANE_STATES] if busy: raise BatchConflict("lane_busy", f"upload batch {busy[0]['id']} is still running") # The approval is re-proved here, immediately before the irreversible act. if not self._preflight.verify_token( batch["preflight_token"], [batch["album"]], allow_partial=batch["allow_partial"] ): self._finish( batch_id, token=batch["version"], state=BatchState.FAILED, error=("stale_preflight", "the library changed after this batch was approved"), ) raise BatchConflict( "stale_preflight", "the library changed after this batch was approved" ) token = self._claim(batch_id, worker_id=worker_id) attempt = self.get(batch_id)["attempt_count"] report_path = Path(self._config.data_dir) / "uploads" / f"{batch_id}-attempt-{attempt}.log" key = self._config.immich_api_key.get_secret_value() if self._config.immich_api_key else "" command = immich_go.build_command( binary=self._config.immich_go_binary, server_url=self._config.immich_server_url, api_key=key, album_name=batch["album_name"], folder=batch["folder"], ) def _stop_requested() -> bool: if cancelled is not None and cancelled(): return True current = self.get(batch_id) return current is None or current["state"] == BatchState.CANCELLING try: result = immich_go.run_upload( command, report_path=report_path, secret=key or None, cancelled=_stop_requested, ) except OSError as error: # uploader vanished between preflight and exec self._finish( batch_id, token=token, state=BatchState.FAILED, error=("uploader_failed", str(error)), ) return self.get(batch_id) if result["cancelled"]: state, error, item_state = BatchState.CANCELLED, None, None elif result["timed_out"]: # Killed mid-flight: the server may already hold some of the files. state = BatchState.UNKNOWN error = ("timeout", "the uploader exceeded its time limit and was stopped") item_state = None elif result["exit_code"] == 0: state, error, item_state = BatchState.SUCCEEDED, None, ItemState.SENT else: state = BatchState.FAILED error = ("uploader_failed", f"immich-go exited with {result['exit_code']}") item_state = ItemState.FAILED # The uploader is done and Immich may already hold every file, but nothing # about that is durable yet — the control point for "accepted, outcome not # recorded" (US07-04). Recovery must answer ``unknown_requires_verification``. maybe_fault(UPLOAD_ACCEPTED) self._finish(batch_id, token=token, state=state, error=error, result=result) if item_state: self._set_items(batch_id, item_state) # The report is the only evidence of what happened per file, so it is read # while it is fresh (US05-03). A parse failure must not lose the batch # outcome that was just recorded; the items simply stay unknown. try: UploadReportService(self._session_factory).ingest(batch_id) except OSError: pass return self.get(batch_id) def cancel(self, batch_id: str) -> dict: """Request a stop. A running attempt drains; a planned batch stops outright.""" batch = self._require(batch_id) if batch["state"] == BatchState.PLANNED: target = BatchState.CANCELLED elif batch["state"] == BatchState.RUNNING: target = BatchState.CANCELLING else: raise BatchError("not_cancellable", f"batch {batch_id} is {batch['state']}") with self._session_factory() as session: row = session.get(UploadBatch, batch_id) row.state = target row.updated_at = _now() if target == BatchState.CANCELLED: row.finished_at = _now() session.commit() return self.get(batch_id) # ── recovery ────────────────────────────────────────────────────────────── def recover(self) -> dict: """Resolve batches whose attempt died with the process. A batch that never started is left ``planned`` and simply runs later. One that was mid-upload cannot be classified from local state — immich-go may have transferred everything before the crash — so it becomes ``unknown_requires_verification`` and releases the lane rather than being retried or declared failed. """ interrupted = 0 with self._session_factory() as session: for row in session.scalars( select(UploadBatch).where(UploadBatch.state.in_(LANE_STATES)) ): row.state = BatchState.UNKNOWN row.error_code = "interrupted" row.error_message = "the uploader process ended without a recorded outcome" row.finished_at = _now() row.updated_at = _now() row.version += 1 interrupted += 1 session.commit() return {"interrupted": interrupted} # ── reads ───────────────────────────────────────────────────────────────── def get(self, batch_id: str) -> dict | None: with self._session_factory() as session: row = session.get(UploadBatch, batch_id) if row is None: return None items = list( session.scalars( select(UploadItem) .where(UploadItem.batch_id == batch_id) .order_by(UploadItem.path) ) ) return _batch_dict(row, items) def list(self) -> list[dict]: with self._session_factory() as session: rows = list(session.scalars(select(UploadBatch).order_by(UploadBatch.created_at))) return [_batch_dict(row, []) for row in rows] def report(self, batch_id: str) -> str: """The raw uploader output kept for this batch, or ``""`` when there is none.""" batch = self._require(batch_id) path = Path(batch["report_path"]) if batch["report_path"] else None if path is None or not path.exists(): return "" return path.read_text(errors="replace") # ── internals ───────────────────────────────────────────────────────────── def _open_batch_for(self, album: str) -> dict | None: with self._session_factory() as session: row = session.scalar( select(UploadBatch).where( UploadBatch.album == album, UploadBatch.state.in_(OPEN_STATES) ) ) return self.get(row.id) if row else None def _claim(self, batch_id: str, *, worker_id: str) -> int: """Take ownership: bump the version (the fencing token) and start an attempt.""" with self._session_factory() as session: row = session.get(UploadBatch, batch_id) row.version += 1 row.attempt_count += 1 row.state = BatchState.RUNNING row.worker_id = worker_id row.error_code = row.error_message = None row.started_at = _now() row.finished_at = None row.updated_at = _now() token = row.version session.commit() return token def _finish( self, batch_id: str, *, token: int, state: str, error: tuple[str, str] | None = None, result: dict | None = None, ) -> None: """Record the outcome, but only for the attempt that still owns the batch.""" values = { "state": state, "finished_at": _now(), "updated_at": _now(), "error_code": error[0] if error else None, "error_message": error[1][:500] if error else None, } if result is not None: values |= { "exit_code": result["exit_code"], "report_path": result["report_path"], "report_bytes": result["report_bytes"], "report_truncated": result["report_truncated"], } with self._session_factory() as session: session.execute( update(UploadBatch) .where(UploadBatch.id == batch_id, UploadBatch.version == token) .values(**values) ) session.commit() def _set_items(self, batch_id: str, state: str) -> None: with self._session_factory() as session: session.execute( update(UploadItem) .where(UploadItem.batch_id == batch_id) .values(state=state, updated_at=_now()) ) session.commit() def _require(self, batch_id: str) -> dict: batch = self.get(batch_id) if batch is None: raise BatchError("not_found", f"unknown upload batch {batch_id!r}") return batch def _batch_dict(row: UploadBatch, items: list[UploadItem]) -> dict: batch = { "id": row.id, "album": row.album, "folder": row.folder, "album_name": row.album_name, "state": row.state, "preflight_token": row.preflight_token, "allow_partial": row.allow_partial, "command": json.loads(row.command or "[]"), "uploader_version": row.uploader_version, "asset_count": row.asset_count, "attempt_count": row.attempt_count, "version": row.version, "worker_id": row.worker_id, "report_path": row.report_path, "report_bytes": row.report_bytes, "report_truncated": row.report_truncated, "exit_code": row.exit_code, "error_code": row.error_code, "error_message": row.error_message, # Parsed report evidence (US05-03): what the uploader said per file, and # whether that evidence is complete enough to be trusted. "parser": row.parser, "parser_version": row.parser_version, "parsed_at": row.parsed_at.isoformat() if row.parsed_at else None, "outcome_state": row.outcome_state, "outcome_counts": json.loads(row.outcome_counts) if row.outcome_counts else None, "report_counts": json.loads(row.report_counts) if row.report_counts else None, # Verification evidence (US05-04). ``stale_bytes`` is the visible warning # that an uploaded file has since been edited. "verified_at": row.verified_at.isoformat() if row.verified_at else None, "stale_bytes": row.stale_bytes, "started_at": row.started_at.isoformat() if row.started_at else None, "finished_at": row.finished_at.isoformat() if row.finished_at else None, "items": [ { "asset_id": item.asset_id, "path": item.path, "sha256": item.sha256, "sha1": item.sha1, "state": item.state, "outcome": item.outcome, "evidence": item.evidence, "outcome_at": item.outcome_at.isoformat() if item.outcome_at else None, "verification": item.verification, "verified_at": item.verified_at.isoformat() if item.verified_at else None, "observed_sha256": item.observed_sha256, "changed_after_upload": item.changed_after_upload, } for item in items ], } # Why this batch may not be (re)started, from the one place that decides it # (US05-04). Carried in the record so the browser can hide an action the server # would refuse instead of re-implementing the policy (US05-05). batch["retry_blockers"] = retry_blockers(batch) return batch