US05-02: Orchestrate Album Upload Batches (#72)
This commit was merged in pull request #72.
This commit is contained in:
@@ -24,7 +24,16 @@ _CHUNK = 1 << 20
|
||||
|
||||
|
||||
def sha256_file(path: Path | str) -> str:
|
||||
digest = hashlib.sha256()
|
||||
return _digest_file(path, hashlib.sha256())
|
||||
|
||||
|
||||
def sha1_file(path: Path | str) -> str:
|
||||
"""SHA-1 of the file bytes. Not an identity hash here — it is the checksum
|
||||
Immich/immich-go use to recognise an asset they already hold (concept §8)."""
|
||||
return _digest_file(path, hashlib.sha1())
|
||||
|
||||
|
||||
def _digest_file(path: Path | str, digest) -> str:
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(_CHUNK), b""):
|
||||
digest.update(chunk)
|
||||
|
||||
424
photo_pipeline/services/upload_batches.py
Normal file
424
photo_pipeline/services/upload_batches.py
Normal file
@@ -0,0 +1,424 @@
|
||||
"""UploadBatchService — one approved album at a time through immich-go (US05-02).
|
||||
|
||||
Preflight (US05-01) proves a scope is safe and issues a token; this service turns
|
||||
that approval into a durable batch and runs it. Upload is the one stage the app
|
||||
cannot undo, so the discipline is:
|
||||
|
||||
- **the approval is re-proved, not remembered.** Before every attempt the batch's
|
||||
preflight token is recomputed from the current library state. Bytes edited after
|
||||
approval, a withdrawn safety decision, or a server that stopped answering all
|
||||
produce a different token and the attempt is refused, never run "optimistically".
|
||||
- **one lane.** A batch can only start while no other batch is running; the album
|
||||
scope of a batch never widens after creation (concept §16 uploader lane).
|
||||
- **cancellation is cooperative and durable.** ``cancel`` writes ``cancelling``;
|
||||
the running attempt observes it through the database, stops the uploader, and
|
||||
records ``cancelled``. A cancelled batch is re-runnable from its own boundary.
|
||||
- **an interrupted attempt is uncertain, not failed.** Immich may have accepted
|
||||
files the app never saw a report for, so ``recover`` marks a batch whose worker
|
||||
vanished ``unknown_requires_verification`` (concept §15) instead of retrying it
|
||||
blindly. Resolving that is US05-04.
|
||||
|
||||
Per-asset upload *results* are deliberately not interpreted here — parsing the
|
||||
report is US05-03. What this story guarantees is that the report exists, is bounded,
|
||||
carries no credential, and that the batch's state is an honest description of what
|
||||
the process did.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from photo_pipeline.config import Config
|
||||
from photo_pipeline.integrations import immich_go
|
||||
from photo_pipeline.models import UploadBatch, UploadItem
|
||||
from photo_pipeline.services.hashing import sha1_file
|
||||
from photo_pipeline.services.uploads import UploadService
|
||||
|
||||
|
||||
class BatchState:
|
||||
PLANNED = "planned"
|
||||
RUNNING = "running"
|
||||
CANCELLING = "cancelling"
|
||||
CANCELLED = "cancelled"
|
||||
SUCCEEDED = "succeeded"
|
||||
FAILED = "failed"
|
||||
# The attempt died without a parsed outcome: the server may hold the files.
|
||||
UNKNOWN = "unknown_requires_verification"
|
||||
|
||||
|
||||
# States that occupy the single uploader lane.
|
||||
LANE_STATES = frozenset({BatchState.RUNNING, BatchState.CANCELLING})
|
||||
# States a batch may be (re)started from. ``unknown_requires_verification`` is not
|
||||
# among them: an uncertain upload must be verified (US05-04), never blindly retried.
|
||||
RUNNABLE_STATES = frozenset({BatchState.PLANNED, BatchState.FAILED, BatchState.CANCELLED})
|
||||
# States whose batch is still the live one for its album.
|
||||
OPEN_STATES = frozenset({BatchState.PLANNED, BatchState.RUNNING, BatchState.CANCELLING})
|
||||
|
||||
class ItemState:
|
||||
PENDING = "pending"
|
||||
# ``sent`` means the batch process exited cleanly, not that Immich confirmed the
|
||||
# asset — the per-item outcome comes from the report in US05-03/US05-04.
|
||||
SENT = "sent"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class BatchError(RuntimeError):
|
||||
"""The request cannot be carried out (unknown batch, wrong state, blocked)."""
|
||||
|
||||
def __init__(self, code: str, message: str) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
|
||||
|
||||
class BatchConflict(BatchError):
|
||||
"""The lane is busy or the approval is stale — retry after resolving it."""
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class UploadBatchService:
|
||||
def __init__(self, session_factory: sessionmaker, *, config: Config) -> None:
|
||||
self._session_factory = session_factory
|
||||
self._config = config
|
||||
self._preflight = UploadService(session_factory, config=config)
|
||||
|
||||
# ── creation ──────────────────────────────────────────────────────────────
|
||||
|
||||
def create(
|
||||
self, albums: list[str] | None = None, *, token: str, allow_partial: bool = False
|
||||
) -> list[dict]:
|
||||
"""Turn a *ready* preflight into one durable batch per album.
|
||||
|
||||
``token`` must be the token of the current preflight for the same scope and
|
||||
policy; anything else means the browser is acting on a stale preview.
|
||||
"""
|
||||
report = self._preflight.preflight(albums, allow_partial=allow_partial)
|
||||
if token != report["token"]:
|
||||
raise BatchConflict(
|
||||
"stale_preflight", "the preflight token does not describe the current state"
|
||||
)
|
||||
if report["state"] != "ready":
|
||||
raise BatchError("not_ready", "the scope has unresolved upload blockers")
|
||||
|
||||
created: list[dict] = []
|
||||
for album in report["albums"]:
|
||||
existing = self._open_batch_for(album["album"])
|
||||
if existing is not None:
|
||||
created.append(existing) # idempotent: one open batch per album
|
||||
continue
|
||||
created.append(self._create_one(album, token=token, allow_partial=allow_partial))
|
||||
return created
|
||||
|
||||
def _create_one(self, album: dict, *, token: str, allow_partial: bool) -> dict:
|
||||
eligible = [asset for asset in album["assets"] if not asset["blockers"]]
|
||||
batch_id = str(uuid.uuid4())
|
||||
command = immich_go.preview_command(
|
||||
binary=self._config.immich_go_binary,
|
||||
server_url=self._config.immich_server_url,
|
||||
album_name=album["album_name"],
|
||||
folder=album["folder"],
|
||||
)
|
||||
with self._session_factory() as session:
|
||||
session.add(
|
||||
UploadBatch(
|
||||
id=batch_id,
|
||||
album=album["album"],
|
||||
folder=album["folder"],
|
||||
album_name=album["album_name"],
|
||||
state=BatchState.PLANNED,
|
||||
preflight_token=token,
|
||||
allow_partial=allow_partial,
|
||||
command=json.dumps(command),
|
||||
uploader_version=immich_go.version(self._config.immich_go_binary),
|
||||
asset_count=len(eligible),
|
||||
)
|
||||
)
|
||||
for asset in eligible:
|
||||
session.add(
|
||||
UploadItem(
|
||||
batch_id=batch_id,
|
||||
asset_id=asset["asset_id"],
|
||||
path=asset["current_path"],
|
||||
sha256=asset["current_sha256"],
|
||||
sha1=sha1_file(asset["current_path"]),
|
||||
state=ItemState.PENDING,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
return self.get(batch_id)
|
||||
|
||||
# ── running ───────────────────────────────────────────────────────────────
|
||||
|
||||
def run(self, batch_id: str, *, worker_id: str = "uploader", cancelled=None) -> dict:
|
||||
"""Run one batch to completion. Blocks for the duration of the upload."""
|
||||
batch = self._require(batch_id)
|
||||
if batch["state"] not in RUNNABLE_STATES:
|
||||
code = (
|
||||
"requires_verification"
|
||||
if batch["state"] == BatchState.UNKNOWN
|
||||
else "not_runnable"
|
||||
)
|
||||
raise BatchError(code, f"batch {batch_id} is {batch['state']}")
|
||||
busy = [row for row in self.list() if row["id"] != batch_id and row["state"] in LANE_STATES]
|
||||
if busy:
|
||||
raise BatchConflict("lane_busy", f"upload batch {busy[0]['id']} is still running")
|
||||
|
||||
# The approval is re-proved here, immediately before the irreversible act.
|
||||
if not self._preflight.verify_token(
|
||||
batch["preflight_token"], [batch["album"]], allow_partial=batch["allow_partial"]
|
||||
):
|
||||
self._finish(
|
||||
batch_id,
|
||||
token=batch["version"],
|
||||
state=BatchState.FAILED,
|
||||
error=("stale_preflight", "the library changed after this batch was approved"),
|
||||
)
|
||||
raise BatchConflict(
|
||||
"stale_preflight", "the library changed after this batch was approved"
|
||||
)
|
||||
|
||||
token = self._claim(batch_id, worker_id=worker_id)
|
||||
attempt = self.get(batch_id)["attempt_count"]
|
||||
report_path = (
|
||||
Path(self._config.data_dir) / "uploads" / f"{batch_id}-attempt-{attempt}.log"
|
||||
)
|
||||
key = self._config.immich_api_key.get_secret_value() if self._config.immich_api_key else ""
|
||||
command = immich_go.build_command(
|
||||
binary=self._config.immich_go_binary,
|
||||
server_url=self._config.immich_server_url,
|
||||
api_key=key,
|
||||
album_name=batch["album_name"],
|
||||
folder=batch["folder"],
|
||||
)
|
||||
|
||||
def _stop_requested() -> bool:
|
||||
if cancelled is not None and cancelled():
|
||||
return True
|
||||
current = self.get(batch_id)
|
||||
return current is None or current["state"] == BatchState.CANCELLING
|
||||
|
||||
try:
|
||||
result = immich_go.run_upload(
|
||||
command,
|
||||
report_path=report_path,
|
||||
secret=key or None,
|
||||
cancelled=_stop_requested,
|
||||
)
|
||||
except OSError as error: # uploader vanished between preflight and exec
|
||||
self._finish(
|
||||
batch_id,
|
||||
token=token,
|
||||
state=BatchState.FAILED,
|
||||
error=("uploader_failed", str(error)),
|
||||
)
|
||||
return self.get(batch_id)
|
||||
|
||||
if result["cancelled"]:
|
||||
state, error, item_state = BatchState.CANCELLED, None, None
|
||||
elif result["timed_out"]:
|
||||
# Killed mid-flight: the server may already hold some of the files.
|
||||
state = BatchState.UNKNOWN
|
||||
error = ("timeout", "the uploader exceeded its time limit and was stopped")
|
||||
item_state = None
|
||||
elif result["exit_code"] == 0:
|
||||
state, error, item_state = BatchState.SUCCEEDED, None, ItemState.SENT
|
||||
else:
|
||||
state = BatchState.FAILED
|
||||
error = ("uploader_failed", f"immich-go exited with {result['exit_code']}")
|
||||
item_state = ItemState.FAILED
|
||||
|
||||
self._finish(batch_id, token=token, state=state, error=error, result=result)
|
||||
if item_state:
|
||||
self._set_items(batch_id, item_state)
|
||||
return self.get(batch_id)
|
||||
|
||||
def cancel(self, batch_id: str) -> dict:
|
||||
"""Request a stop. A running attempt drains; a planned batch stops outright."""
|
||||
batch = self._require(batch_id)
|
||||
if batch["state"] == BatchState.PLANNED:
|
||||
target = BatchState.CANCELLED
|
||||
elif batch["state"] == BatchState.RUNNING:
|
||||
target = BatchState.CANCELLING
|
||||
else:
|
||||
raise BatchError("not_cancellable", f"batch {batch_id} is {batch['state']}")
|
||||
with self._session_factory() as session:
|
||||
row = session.get(UploadBatch, batch_id)
|
||||
row.state = target
|
||||
row.updated_at = _now()
|
||||
if target == BatchState.CANCELLED:
|
||||
row.finished_at = _now()
|
||||
session.commit()
|
||||
return self.get(batch_id)
|
||||
|
||||
# ── recovery ──────────────────────────────────────────────────────────────
|
||||
|
||||
def recover(self) -> dict:
|
||||
"""Resolve batches whose attempt died with the process.
|
||||
|
||||
A batch that never started is left ``planned`` and simply runs later. One
|
||||
that was mid-upload cannot be classified from local state — immich-go may
|
||||
have transferred everything before the crash — so it becomes
|
||||
``unknown_requires_verification`` and releases the lane rather than being
|
||||
retried or declared failed.
|
||||
"""
|
||||
interrupted = 0
|
||||
with self._session_factory() as session:
|
||||
for row in session.scalars(
|
||||
select(UploadBatch).where(UploadBatch.state.in_(LANE_STATES))
|
||||
):
|
||||
row.state = BatchState.UNKNOWN
|
||||
row.error_code = "interrupted"
|
||||
row.error_message = "the uploader process ended without a recorded outcome"
|
||||
row.finished_at = _now()
|
||||
row.updated_at = _now()
|
||||
row.version += 1
|
||||
interrupted += 1
|
||||
session.commit()
|
||||
return {"interrupted": interrupted}
|
||||
|
||||
# ── reads ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def get(self, batch_id: str) -> dict | None:
|
||||
with self._session_factory() as session:
|
||||
row = session.get(UploadBatch, batch_id)
|
||||
if row is None:
|
||||
return None
|
||||
items = list(
|
||||
session.scalars(
|
||||
select(UploadItem)
|
||||
.where(UploadItem.batch_id == batch_id)
|
||||
.order_by(UploadItem.path)
|
||||
)
|
||||
)
|
||||
return _batch_dict(row, items)
|
||||
|
||||
def list(self) -> list[dict]:
|
||||
with self._session_factory() as session:
|
||||
rows = list(session.scalars(select(UploadBatch).order_by(UploadBatch.created_at)))
|
||||
return [_batch_dict(row, []) for row in rows]
|
||||
|
||||
def report(self, batch_id: str) -> str:
|
||||
"""The raw uploader output kept for this batch, or ``""`` when there is none."""
|
||||
batch = self._require(batch_id)
|
||||
path = Path(batch["report_path"]) if batch["report_path"] else None
|
||||
if path is None or not path.exists():
|
||||
return ""
|
||||
return path.read_text(errors="replace")
|
||||
|
||||
# ── internals ─────────────────────────────────────────────────────────────
|
||||
|
||||
def _open_batch_for(self, album: str) -> dict | None:
|
||||
with self._session_factory() as session:
|
||||
row = session.scalar(
|
||||
select(UploadBatch).where(
|
||||
UploadBatch.album == album, UploadBatch.state.in_(OPEN_STATES)
|
||||
)
|
||||
)
|
||||
return self.get(row.id) if row else None
|
||||
|
||||
def _claim(self, batch_id: str, *, worker_id: str) -> int:
|
||||
"""Take ownership: bump the version (the fencing token) and start an attempt."""
|
||||
with self._session_factory() as session:
|
||||
row = session.get(UploadBatch, batch_id)
|
||||
row.version += 1
|
||||
row.attempt_count += 1
|
||||
row.state = BatchState.RUNNING
|
||||
row.worker_id = worker_id
|
||||
row.error_code = row.error_message = None
|
||||
row.started_at = _now()
|
||||
row.finished_at = None
|
||||
row.updated_at = _now()
|
||||
token = row.version
|
||||
session.commit()
|
||||
return token
|
||||
|
||||
def _finish(
|
||||
self,
|
||||
batch_id: str,
|
||||
*,
|
||||
token: int,
|
||||
state: str,
|
||||
error: tuple[str, str] | None = None,
|
||||
result: dict | None = None,
|
||||
) -> None:
|
||||
"""Record the outcome, but only for the attempt that still owns the batch."""
|
||||
values = {
|
||||
"state": state,
|
||||
"finished_at": _now(),
|
||||
"updated_at": _now(),
|
||||
"error_code": error[0] if error else None,
|
||||
"error_message": error[1][:500] if error else None,
|
||||
}
|
||||
if result is not None:
|
||||
values |= {
|
||||
"exit_code": result["exit_code"],
|
||||
"report_path": result["report_path"],
|
||||
"report_bytes": result["report_bytes"],
|
||||
"report_truncated": result["report_truncated"],
|
||||
}
|
||||
with self._session_factory() as session:
|
||||
session.execute(
|
||||
update(UploadBatch)
|
||||
.where(UploadBatch.id == batch_id, UploadBatch.version == token)
|
||||
.values(**values)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
def _set_items(self, batch_id: str, state: str) -> None:
|
||||
with self._session_factory() as session:
|
||||
session.execute(
|
||||
update(UploadItem)
|
||||
.where(UploadItem.batch_id == batch_id)
|
||||
.values(state=state, updated_at=_now())
|
||||
)
|
||||
session.commit()
|
||||
|
||||
def _require(self, batch_id: str) -> dict:
|
||||
batch = self.get(batch_id)
|
||||
if batch is None:
|
||||
raise BatchError("not_found", f"unknown upload batch {batch_id!r}")
|
||||
return batch
|
||||
|
||||
|
||||
def _batch_dict(row: UploadBatch, items: list[UploadItem]) -> dict:
|
||||
return {
|
||||
"id": row.id,
|
||||
"album": row.album,
|
||||
"folder": row.folder,
|
||||
"album_name": row.album_name,
|
||||
"state": row.state,
|
||||
"preflight_token": row.preflight_token,
|
||||
"allow_partial": row.allow_partial,
|
||||
"command": json.loads(row.command or "[]"),
|
||||
"uploader_version": row.uploader_version,
|
||||
"asset_count": row.asset_count,
|
||||
"attempt_count": row.attempt_count,
|
||||
"version": row.version,
|
||||
"worker_id": row.worker_id,
|
||||
"report_path": row.report_path,
|
||||
"report_bytes": row.report_bytes,
|
||||
"report_truncated": row.report_truncated,
|
||||
"exit_code": row.exit_code,
|
||||
"error_code": row.error_code,
|
||||
"error_message": row.error_message,
|
||||
"started_at": row.started_at.isoformat() if row.started_at else None,
|
||||
"finished_at": row.finished_at.isoformat() if row.finished_at else None,
|
||||
"items": [
|
||||
{
|
||||
"asset_id": item.asset_id,
|
||||
"path": item.path,
|
||||
"sha256": item.sha256,
|
||||
"sha1": item.sha1,
|
||||
"state": item.state,
|
||||
}
|
||||
for item in items
|
||||
],
|
||||
}
|
||||
Reference in New Issue
Block a user