462 lines
20 KiB
Python
462 lines
20 KiB
Python
"""UploadBatchService — one approved album at a time through immich-go (US05-02).
|
|
|
|
Preflight (US05-01) proves a scope is safe and issues a token; this service turns
|
|
that approval into a durable batch and runs it. Upload is the one stage the app
|
|
cannot undo, so the discipline is:
|
|
|
|
- **the approval is re-proved, not remembered.** Before every attempt the batch's
|
|
preflight token is recomputed from the current library state. Bytes edited after
|
|
approval, a withdrawn safety decision, or a server that stopped answering all
|
|
produce a different token and the attempt is refused, never run "optimistically".
|
|
- **one lane.** A batch can only start while no other batch is running; the album
|
|
scope of a batch never widens after creation (concept §16 uploader lane).
|
|
- **cancellation is cooperative and durable.** ``cancel`` writes ``cancelling``;
|
|
the running attempt observes it through the database, stops the uploader, and
|
|
records ``cancelled``. A cancelled batch is re-runnable from its own boundary.
|
|
- **an interrupted attempt is uncertain, not failed.** Immich may have accepted
|
|
files the app never saw a report for, so ``recover`` marks a batch whose worker
|
|
vanished ``unknown_requires_verification`` (concept §15) instead of retrying it
|
|
blindly. :func:`~photo_pipeline.services.upload_verification.retry_blockers`
|
|
(US05-04) is what decides whether an attempt may start at all.
|
|
|
|
Per-asset upload *results* are not interpreted here: after the attempt ends the
|
|
report is handed to :class:`~photo_pipeline.services.upload_reports.
|
|
UploadReportService` (US05-03), which classifies every file. The batch ``state``
|
|
stays an honest description of what the *process* did; ``outcome_state`` says
|
|
whether the report's evidence is complete enough to trust.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import select, update
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
from photo_pipeline.config import Config
|
|
from photo_pipeline.faults import UPLOAD_ACCEPTED, maybe_fault
|
|
from photo_pipeline.integrations import immich_go
|
|
from photo_pipeline.models import UploadBatch, UploadItem
|
|
from photo_pipeline.services.hashing import sha1_file
|
|
from photo_pipeline.services.upload_reports import UploadReportService
|
|
from photo_pipeline.services.upload_verification import retry_blockers
|
|
from photo_pipeline.services.uploads import UploadService
|
|
|
|
|
|
class BatchState:
|
|
PLANNED = "planned"
|
|
RUNNING = "running"
|
|
CANCELLING = "cancelling"
|
|
CANCELLED = "cancelled"
|
|
SUCCEEDED = "succeeded"
|
|
FAILED = "failed"
|
|
# The attempt died without a parsed outcome: the server may hold the files.
|
|
UNKNOWN = "unknown_requires_verification"
|
|
|
|
|
|
# States that occupy the single uploader lane.
|
|
LANE_STATES = frozenset({BatchState.RUNNING, BatchState.CANCELLING})
|
|
# States a batch may be (re)started from. ``unknown_requires_verification`` is not
|
|
# among them: an uncertain upload must be verified (US05-04), never blindly retried.
|
|
RUNNABLE_STATES = frozenset({BatchState.PLANNED, BatchState.FAILED, BatchState.CANCELLED})
|
|
# States whose batch is still the live one for its album.
|
|
OPEN_STATES = frozenset({BatchState.PLANNED, BatchState.RUNNING, BatchState.CANCELLING})
|
|
|
|
|
|
class ItemState:
|
|
PENDING = "pending"
|
|
# ``sent`` means the batch process exited cleanly, not that Immich confirmed the
|
|
# asset — the per-item outcome comes from the report in US05-03/US05-04.
|
|
SENT = "sent"
|
|
FAILED = "failed"
|
|
|
|
|
|
class BatchError(RuntimeError):
|
|
"""The request cannot be carried out (unknown batch, wrong state, blocked)."""
|
|
|
|
def __init__(self, code: str, message: str) -> None:
|
|
super().__init__(message)
|
|
self.code = code
|
|
|
|
|
|
class BatchConflict(BatchError):
|
|
"""The lane is busy or the approval is stale — retry after resolving it."""
|
|
|
|
|
|
def _now() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
class UploadBatchService:
|
|
def __init__(self, session_factory: sessionmaker, *, config: Config) -> None:
|
|
self._session_factory = session_factory
|
|
self._config = config
|
|
self._preflight = UploadService(session_factory, config=config)
|
|
|
|
# ── creation ──────────────────────────────────────────────────────────────
|
|
|
|
def create(
|
|
self, albums: list[str] | None = None, *, token: str, allow_partial: bool = False
|
|
) -> list[dict]:
|
|
"""Turn a *ready* preflight into one durable batch per album.
|
|
|
|
``token`` must be the token of the current preflight for the same scope and
|
|
policy; anything else means the browser is acting on a stale preview.
|
|
"""
|
|
report = self._preflight.preflight(albums, allow_partial=allow_partial)
|
|
if token != report["token"]:
|
|
raise BatchConflict(
|
|
"stale_preflight", "the preflight token does not describe the current state"
|
|
)
|
|
if report["state"] != "ready":
|
|
raise BatchError("not_ready", "the scope has unresolved upload blockers")
|
|
|
|
created: list[dict] = []
|
|
for album in report["albums"]:
|
|
existing = self._open_batch_for(album["album"])
|
|
if existing is not None:
|
|
created.append(existing) # idempotent: one open batch per album
|
|
continue
|
|
created.append(self._create_one(album, token=token, allow_partial=allow_partial))
|
|
return created
|
|
|
|
def _create_one(self, album: dict, *, token: str, allow_partial: bool) -> dict:
|
|
eligible = [asset for asset in album["assets"] if not asset["blockers"]]
|
|
batch_id = str(uuid.uuid4())
|
|
command = immich_go.preview_command(
|
|
binary=self._config.immich_go_binary,
|
|
server_url=self._config.immich_server_url,
|
|
album_name=album["album_name"],
|
|
folder=album["folder"],
|
|
)
|
|
with self._session_factory() as session:
|
|
session.add(
|
|
UploadBatch(
|
|
id=batch_id,
|
|
album=album["album"],
|
|
folder=album["folder"],
|
|
album_name=album["album_name"],
|
|
state=BatchState.PLANNED,
|
|
preflight_token=token,
|
|
allow_partial=allow_partial,
|
|
command=json.dumps(command),
|
|
uploader_version=immich_go.version(self._config.immich_go_binary),
|
|
asset_count=len(eligible),
|
|
)
|
|
)
|
|
for asset in eligible:
|
|
session.add(
|
|
UploadItem(
|
|
batch_id=batch_id,
|
|
asset_id=asset["asset_id"],
|
|
path=asset["current_path"],
|
|
sha256=asset["current_sha256"],
|
|
sha1=sha1_file(asset["current_path"]),
|
|
state=ItemState.PENDING,
|
|
)
|
|
)
|
|
session.commit()
|
|
return self.get(batch_id)
|
|
|
|
# ── running ───────────────────────────────────────────────────────────────
|
|
|
|
def run(self, batch_id: str, *, worker_id: str = "uploader", cancelled=None) -> dict:
|
|
"""Run one batch to completion. Blocks for the duration of the upload."""
|
|
batch = self._require(batch_id)
|
|
# Retry policy (US05-04): a safe failure may run again, an uncertain outcome
|
|
# or bytes edited after upload may not.
|
|
blocked = retry_blockers(batch)
|
|
if blocked:
|
|
raise BatchError(blocked[0]["code"], blocked[0]["message"])
|
|
busy = [row for row in self.list() if row["id"] != batch_id and row["state"] in LANE_STATES]
|
|
if busy:
|
|
raise BatchConflict("lane_busy", f"upload batch {busy[0]['id']} is still running")
|
|
|
|
# The approval is re-proved here, immediately before the irreversible act.
|
|
if not self._preflight.verify_token(
|
|
batch["preflight_token"], [batch["album"]], allow_partial=batch["allow_partial"]
|
|
):
|
|
self._finish(
|
|
batch_id,
|
|
token=batch["version"],
|
|
state=BatchState.FAILED,
|
|
error=("stale_preflight", "the library changed after this batch was approved"),
|
|
)
|
|
raise BatchConflict(
|
|
"stale_preflight", "the library changed after this batch was approved"
|
|
)
|
|
|
|
token = self._claim(batch_id, worker_id=worker_id)
|
|
attempt = self.get(batch_id)["attempt_count"]
|
|
report_path = Path(self._config.data_dir) / "uploads" / f"{batch_id}-attempt-{attempt}.log"
|
|
key = self._config.immich_api_key.get_secret_value() if self._config.immich_api_key else ""
|
|
command = immich_go.build_command(
|
|
binary=self._config.immich_go_binary,
|
|
server_url=self._config.immich_server_url,
|
|
api_key=key,
|
|
album_name=batch["album_name"],
|
|
folder=batch["folder"],
|
|
)
|
|
|
|
def _stop_requested() -> bool:
|
|
if cancelled is not None and cancelled():
|
|
return True
|
|
current = self.get(batch_id)
|
|
return current is None or current["state"] == BatchState.CANCELLING
|
|
|
|
try:
|
|
result = immich_go.run_upload(
|
|
command,
|
|
report_path=report_path,
|
|
secret=key or None,
|
|
cancelled=_stop_requested,
|
|
)
|
|
except OSError as error: # uploader vanished between preflight and exec
|
|
self._finish(
|
|
batch_id,
|
|
token=token,
|
|
state=BatchState.FAILED,
|
|
error=("uploader_failed", str(error)),
|
|
)
|
|
return self.get(batch_id)
|
|
|
|
if result["cancelled"]:
|
|
state, error, item_state = BatchState.CANCELLED, None, None
|
|
elif result["timed_out"]:
|
|
# Killed mid-flight: the server may already hold some of the files.
|
|
state = BatchState.UNKNOWN
|
|
error = ("timeout", "the uploader exceeded its time limit and was stopped")
|
|
item_state = None
|
|
elif result["exit_code"] == 0:
|
|
state, error, item_state = BatchState.SUCCEEDED, None, ItemState.SENT
|
|
else:
|
|
state = BatchState.FAILED
|
|
error = ("uploader_failed", f"immich-go exited with {result['exit_code']}")
|
|
item_state = ItemState.FAILED
|
|
|
|
# The uploader is done and Immich may already hold every file, but nothing
|
|
# about that is durable yet — the control point for "accepted, outcome not
|
|
# recorded" (US07-04). Recovery must answer ``unknown_requires_verification``.
|
|
maybe_fault(UPLOAD_ACCEPTED)
|
|
self._finish(batch_id, token=token, state=state, error=error, result=result)
|
|
if item_state:
|
|
self._set_items(batch_id, item_state)
|
|
# The report is the only evidence of what happened per file, so it is read
|
|
# while it is fresh (US05-03). A parse failure must not lose the batch
|
|
# outcome that was just recorded; the items simply stay unknown.
|
|
try:
|
|
UploadReportService(self._session_factory).ingest(batch_id)
|
|
except OSError:
|
|
pass
|
|
return self.get(batch_id)
|
|
|
|
def cancel(self, batch_id: str) -> dict:
|
|
"""Request a stop. A running attempt drains; a planned batch stops outright."""
|
|
batch = self._require(batch_id)
|
|
if batch["state"] == BatchState.PLANNED:
|
|
target = BatchState.CANCELLED
|
|
elif batch["state"] == BatchState.RUNNING:
|
|
target = BatchState.CANCELLING
|
|
else:
|
|
raise BatchError("not_cancellable", f"batch {batch_id} is {batch['state']}")
|
|
with self._session_factory() as session:
|
|
row = session.get(UploadBatch, batch_id)
|
|
row.state = target
|
|
row.updated_at = _now()
|
|
if target == BatchState.CANCELLED:
|
|
row.finished_at = _now()
|
|
session.commit()
|
|
return self.get(batch_id)
|
|
|
|
# ── recovery ──────────────────────────────────────────────────────────────
|
|
|
|
def recover(self) -> dict:
|
|
"""Resolve batches whose attempt died with the process.
|
|
|
|
A batch that never started is left ``planned`` and simply runs later. One
|
|
that was mid-upload cannot be classified from local state — immich-go may
|
|
have transferred everything before the crash — so it becomes
|
|
``unknown_requires_verification`` and releases the lane rather than being
|
|
retried or declared failed.
|
|
"""
|
|
interrupted = 0
|
|
with self._session_factory() as session:
|
|
for row in session.scalars(
|
|
select(UploadBatch).where(UploadBatch.state.in_(LANE_STATES))
|
|
):
|
|
row.state = BatchState.UNKNOWN
|
|
row.error_code = "interrupted"
|
|
row.error_message = "the uploader process ended without a recorded outcome"
|
|
row.finished_at = _now()
|
|
row.updated_at = _now()
|
|
row.version += 1
|
|
interrupted += 1
|
|
session.commit()
|
|
return {"interrupted": interrupted}
|
|
|
|
# ── reads ─────────────────────────────────────────────────────────────────
|
|
|
|
def get(self, batch_id: str) -> dict | None:
|
|
with self._session_factory() as session:
|
|
row = session.get(UploadBatch, batch_id)
|
|
if row is None:
|
|
return None
|
|
items = list(
|
|
session.scalars(
|
|
select(UploadItem)
|
|
.where(UploadItem.batch_id == batch_id)
|
|
.order_by(UploadItem.path)
|
|
)
|
|
)
|
|
return _batch_dict(row, items)
|
|
|
|
def list(self) -> list[dict]:
|
|
with self._session_factory() as session:
|
|
rows = list(session.scalars(select(UploadBatch).order_by(UploadBatch.created_at)))
|
|
return [_batch_dict(row, []) for row in rows]
|
|
|
|
def report(self, batch_id: str) -> str:
|
|
"""The raw uploader output kept for this batch, or ``""`` when there is none."""
|
|
batch = self._require(batch_id)
|
|
path = Path(batch["report_path"]) if batch["report_path"] else None
|
|
if path is None or not path.exists():
|
|
return ""
|
|
return path.read_text(errors="replace")
|
|
|
|
# ── internals ─────────────────────────────────────────────────────────────
|
|
|
|
def _open_batch_for(self, album: str) -> dict | None:
|
|
with self._session_factory() as session:
|
|
row = session.scalar(
|
|
select(UploadBatch).where(
|
|
UploadBatch.album == album, UploadBatch.state.in_(OPEN_STATES)
|
|
)
|
|
)
|
|
return self.get(row.id) if row else None
|
|
|
|
def _claim(self, batch_id: str, *, worker_id: str) -> int:
|
|
"""Take ownership: bump the version (the fencing token) and start an attempt."""
|
|
with self._session_factory() as session:
|
|
row = session.get(UploadBatch, batch_id)
|
|
row.version += 1
|
|
row.attempt_count += 1
|
|
row.state = BatchState.RUNNING
|
|
row.worker_id = worker_id
|
|
row.error_code = row.error_message = None
|
|
row.started_at = _now()
|
|
row.finished_at = None
|
|
row.updated_at = _now()
|
|
token = row.version
|
|
session.commit()
|
|
return token
|
|
|
|
def _finish(
|
|
self,
|
|
batch_id: str,
|
|
*,
|
|
token: int,
|
|
state: str,
|
|
error: tuple[str, str] | None = None,
|
|
result: dict | None = None,
|
|
) -> None:
|
|
"""Record the outcome, but only for the attempt that still owns the batch."""
|
|
values = {
|
|
"state": state,
|
|
"finished_at": _now(),
|
|
"updated_at": _now(),
|
|
"error_code": error[0] if error else None,
|
|
"error_message": error[1][:500] if error else None,
|
|
}
|
|
if result is not None:
|
|
values |= {
|
|
"exit_code": result["exit_code"],
|
|
"report_path": result["report_path"],
|
|
"report_bytes": result["report_bytes"],
|
|
"report_truncated": result["report_truncated"],
|
|
}
|
|
with self._session_factory() as session:
|
|
session.execute(
|
|
update(UploadBatch)
|
|
.where(UploadBatch.id == batch_id, UploadBatch.version == token)
|
|
.values(**values)
|
|
)
|
|
session.commit()
|
|
|
|
def _set_items(self, batch_id: str, state: str) -> None:
|
|
with self._session_factory() as session:
|
|
session.execute(
|
|
update(UploadItem)
|
|
.where(UploadItem.batch_id == batch_id)
|
|
.values(state=state, updated_at=_now())
|
|
)
|
|
session.commit()
|
|
|
|
def _require(self, batch_id: str) -> dict:
|
|
batch = self.get(batch_id)
|
|
if batch is None:
|
|
raise BatchError("not_found", f"unknown upload batch {batch_id!r}")
|
|
return batch
|
|
|
|
|
|
def _batch_dict(row: UploadBatch, items: list[UploadItem]) -> dict:
|
|
batch = {
|
|
"id": row.id,
|
|
"album": row.album,
|
|
"folder": row.folder,
|
|
"album_name": row.album_name,
|
|
"state": row.state,
|
|
"preflight_token": row.preflight_token,
|
|
"allow_partial": row.allow_partial,
|
|
"command": json.loads(row.command or "[]"),
|
|
"uploader_version": row.uploader_version,
|
|
"asset_count": row.asset_count,
|
|
"attempt_count": row.attempt_count,
|
|
"version": row.version,
|
|
"worker_id": row.worker_id,
|
|
"report_path": row.report_path,
|
|
"report_bytes": row.report_bytes,
|
|
"report_truncated": row.report_truncated,
|
|
"exit_code": row.exit_code,
|
|
"error_code": row.error_code,
|
|
"error_message": row.error_message,
|
|
# Parsed report evidence (US05-03): what the uploader said per file, and
|
|
# whether that evidence is complete enough to be trusted.
|
|
"parser": row.parser,
|
|
"parser_version": row.parser_version,
|
|
"parsed_at": row.parsed_at.isoformat() if row.parsed_at else None,
|
|
"outcome_state": row.outcome_state,
|
|
"outcome_counts": json.loads(row.outcome_counts) if row.outcome_counts else None,
|
|
"report_counts": json.loads(row.report_counts) if row.report_counts else None,
|
|
# Verification evidence (US05-04). ``stale_bytes`` is the visible warning
|
|
# that an uploaded file has since been edited.
|
|
"verified_at": row.verified_at.isoformat() if row.verified_at else None,
|
|
"stale_bytes": row.stale_bytes,
|
|
"started_at": row.started_at.isoformat() if row.started_at else None,
|
|
"finished_at": row.finished_at.isoformat() if row.finished_at else None,
|
|
"items": [
|
|
{
|
|
"asset_id": item.asset_id,
|
|
"path": item.path,
|
|
"sha256": item.sha256,
|
|
"sha1": item.sha1,
|
|
"state": item.state,
|
|
"outcome": item.outcome,
|
|
"evidence": item.evidence,
|
|
"outcome_at": item.outcome_at.isoformat() if item.outcome_at else None,
|
|
"verification": item.verification,
|
|
"verified_at": item.verified_at.isoformat() if item.verified_at else None,
|
|
"observed_sha256": item.observed_sha256,
|
|
"changed_after_upload": item.changed_after_upload,
|
|
}
|
|
for item in items
|
|
],
|
|
}
|
|
# Why this batch may not be (re)started, from the one place that decides it
|
|
# (US05-04). Carried in the record so the browser can hide an action the server
|
|
# would refuse instead of re-implementing the policy (US05-05).
|
|
batch["retry_blockers"] = retry_blockers(batch)
|
|
return batch
|