US05-02: Orchestrate Album Upload Batches (#72)

photo_pipeline/integrations/immich_go.py

@@ -9,20 +9,42 @@ the command that would actually run.
 
 Server reachability uses ``/api/server/ping`` through stdlib ``urllib`` — the app
 has no HTTP client dependency and this is one request.
+
+:func:`run_upload` is the only place the uploader is actually executed. It never
+uses a shell (the argument list goes straight to ``execve``, so no path or album
+name can be interpreted), streams the report to a file with a byte cap so a chatty
+or looping uploader cannot fill the disk, scrubs the API key out of anything the
+process echoes back, and polls a cancellation callback so a running upload can be
+stopped without killing the worker.
 """
 
 from __future__ import annotations
 
 import json
+import os
 import shutil
+import signal
 import subprocess
+import threading
+import time
 import urllib.error
 import urllib.request
+from collections.abc import Callable
 from pathlib import Path
 
 REDACTED = "***"
 PING_PATH = "/api/server/ping"
 PING_TIMEOUT_SECONDS = 5.0
+# Reports are kept in full up to this size; beyond it the tail is dropped and the
+# result is flagged truncated rather than growing without bound (concept §17).
+MAX_REPORT_BYTES = 4_000_000
+UPLOAD_TIMEOUT_SECONDS = 6 * 60 * 60
+POLL_SECONDS = 0.05
+# Grace period between asking the uploader to stop and killing it.
+TERMINATE_GRACE_SECONDS = 10.0
+# How long to wait for the last output after the process is gone. A child that
+# outlived its parent can still hold the pipe; the report is not worth hanging for.
+DRAIN_SECONDS = 2.0
 
 
 def find_binary(binary: str = "immich-go") -> str | None:
@@ -92,6 +114,103 @@ def redact(command: list[str]) -> list[str]:
     return [f"--api-key={REDACTED}" if arg.startswith("--api-key=") else arg for arg in command]
 
 
+def run_upload(
+    command: list[str],
+    *,
+    report_path: Path | str,
+    secret: str | None = None,
+    max_report_bytes: int | None = None,  # resolved at call time; see MAX_REPORT_BYTES
+    cancelled: Callable[[], bool] | None = None,
+    timeout: float = UPLOAD_TIMEOUT_SECONDS,
+) -> dict:
+    """Run one upload and return its outcome.
+
+    ``{"exit_code", "cancelled", "timed_out", "report_path", "report_bytes",
+    "report_truncated"}``. Output is streamed to ``report_path`` with ``secret``
+    masked and the file capped at ``max_report_bytes``; the pipe keeps being drained
+    after the cap so the child never blocks on a full buffer. ``cancelled`` is polled
+    while the process runs: when it returns true the uploader is asked to stop, then
+    killed if it does not.
+    """
+    report_path = Path(report_path)
+    report_path.parent.mkdir(parents=True, exist_ok=True)
+    max_report_bytes = MAX_REPORT_BYTES if max_report_bytes is None else max_report_bytes
+    needle = (secret or "").encode() or None
+    written = 0
+    truncated = False
+
+    process = subprocess.Popen(  # noqa: S603 — argv list, never a shell string
+        command,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        shell=False,
+        # Own process group: stopping the upload must stop whatever the uploader
+        # spawned too, not leave orphans holding the pipe open.
+        start_new_session=True,
+    )
+
+    def _drain() -> None:
+        nonlocal written, truncated
+        with open(report_path, "wb") as report:
+            for line in process.stdout:  # line granularity keeps the mask reliable
+                if needle:
+                    line = line.replace(needle, REDACTED.encode())
+                if written >= max_report_bytes:
+                    truncated = True
+                    continue  # keep draining the pipe, stop growing the file
+                room = max_report_bytes - written
+                report.write(line[:room])
+                written += min(len(line), room)
+                truncated = truncated or len(line) > room
+                report.flush()
+
+    reader = threading.Thread(target=_drain, daemon=True)
+    reader.start()
+
+    stopped = timed_out = False
+    deadline = time.monotonic() + timeout
+    while process.poll() is None:
+        if cancelled is not None and cancelled():
+            stopped = True
+        elif time.monotonic() >= deadline:
+            timed_out = True
+        if stopped or timed_out:
+            _stop(process)
+            break
+        time.sleep(POLL_SECONDS)
+
+    exit_code = process.wait()
+    reader.join(timeout=DRAIN_SECONDS)
+    if process.stdout is not None:
+        process.stdout.close()
+    return {
+        "exit_code": exit_code,
+        "cancelled": stopped,
+        "timed_out": timed_out,
+        "report_path": str(report_path),
+        "report_bytes": written,
+        "report_truncated": truncated,
+    }
+
+
+def _stop(process: subprocess.Popen) -> None:
+    """Ask the uploader's whole process group to stop, then kill what remains."""
+    _signal_group(process, signal.SIGTERM)
+    try:
+        process.wait(timeout=TERMINATE_GRACE_SECONDS)
+    except subprocess.TimeoutExpired:
+        _signal_group(process, signal.SIGKILL)
+        process.wait()
+
+
+def _signal_group(process: subprocess.Popen, sig: int) -> None:
+    try:
+        os.killpg(os.getpgid(process.pid), sig)
+    except (ProcessLookupError, PermissionError, OSError):
+        # No group (already reaped, or a platform without them): signal the child.
+        process.send_signal(sig)
+
+
 def preview_command(
     *, binary: str, server_url: str, album_name: str, folder: Path | str
 ) -> list[str]: