296 lines
12 KiB
Python
296 lines
12 KiB
Python
"""immich-go adapter: binary discovery, version, and redacted command preview.
|
|
|
|
Upload is the only stage that needs credentials, so this module is also the single
|
|
place that knows an API key exists. It never returns, logs, or renders the secret:
|
|
:func:`build_command` produces the real argument list for the uploader, and
|
|
:func:`redact` produces the copy that is safe for the API, the browser, and the
|
|
activity log. Both come from the same builder so the preview can never drift from
|
|
the command that would actually run.
|
|
|
|
Server reachability uses ``/api/server/ping`` through stdlib ``urllib`` — the app
|
|
has no HTTP client dependency and this is one request. :func:`bulk_upload_check`
|
|
uses the same client to ask Immich which uploaded bytes it already holds, which is
|
|
the authoritative evidence behind upload verification (US05-04).
|
|
|
|
:func:`run_upload` is the only place the uploader is actually executed. It never
|
|
uses a shell (the argument list goes straight to ``execve``, so no path or album
|
|
name can be interpreted), streams the report to a file with a byte cap so a chatty
|
|
or looping uploader cannot fill the disk, scrubs the API key out of anything the
|
|
process echoes back, and polls a cancellation callback so a running upload can be
|
|
stopped without killing the worker.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
import signal
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from collections.abc import Callable
|
|
from pathlib import Path
|
|
|
|
REDACTED = "***"
|
|
PING_PATH = "/api/server/ping"
|
|
PING_TIMEOUT_SECONDS = 5.0
|
|
# Immich's own deduplication endpoint: the authoritative answer to "do you already
|
|
# have these exact bytes?" used to verify uncertain uploads (US05-04).
|
|
BULK_CHECK_PATH = "/api/assets/bulk-upload-check"
|
|
CHECK_TIMEOUT_SECONDS = 30.0
|
|
CHECK_BATCH_SIZE = 500
|
|
# Reports are kept in full up to this size; beyond it the tail is dropped and the
|
|
# result is flagged truncated rather than growing without bound (concept §17).
|
|
MAX_REPORT_BYTES = 4_000_000
|
|
UPLOAD_TIMEOUT_SECONDS = 6 * 60 * 60
|
|
POLL_SECONDS = 0.05
|
|
# Grace period between asking the uploader to stop and killing it.
|
|
TERMINATE_GRACE_SECONDS = 10.0
|
|
# How long to wait for the last output after the process is gone. A child that
|
|
# outlived its parent can still hold the pipe; the report is not worth hanging for.
|
|
DRAIN_SECONDS = 2.0
|
|
|
|
|
|
def find_binary(binary: str = "immich-go") -> str | None:
|
|
"""Absolute path of the uploader, or ``None`` when it is not installed."""
|
|
return shutil.which(binary)
|
|
|
|
|
|
def version(binary: str = "immich-go") -> str | None:
|
|
"""Reported uploader version, or ``None`` when it is missing or unusable.
|
|
|
|
The version is persisted with every batch (concept §8) because immich-go's
|
|
flags and report text change between releases.
|
|
"""
|
|
path = find_binary(binary)
|
|
if path is None:
|
|
return None
|
|
try:
|
|
result = subprocess.run([path, "--version"], capture_output=True, text=True, timeout=30)
|
|
except (OSError, subprocess.SubprocessError):
|
|
return None
|
|
output = (result.stdout or result.stderr or "").strip()
|
|
return output.splitlines()[0].strip() if output else None
|
|
|
|
|
|
def ping(server_url: str, *, timeout: float = PING_TIMEOUT_SECONDS) -> tuple[bool, str | None]:
|
|
"""``(reachable, detail)`` for the configured Immich server.
|
|
|
|
A reachable Immich answers ``{"res": "pong"}``. Anything else — wrong host, no
|
|
Immich, HTTP error — is a blocker with a short human detail. The detail never
|
|
carries the URL's credentials because the key travels in a header, not the URL.
|
|
"""
|
|
if not server_url:
|
|
return False, "no server URL configured"
|
|
url = server_url.rstrip("/") + PING_PATH
|
|
try:
|
|
with urllib.request.urlopen(url, timeout=timeout) as response: # noqa: S310
|
|
payload = json.loads(response.read().decode("utf-8") or "{}")
|
|
except (urllib.error.URLError, OSError, ValueError, TimeoutError) as error:
|
|
return False, f"{type(error).__name__}: {error}"
|
|
if payload.get("res") == "pong":
|
|
return True, None
|
|
return False, "server did not answer with pong"
|
|
|
|
|
|
def bulk_upload_check(
|
|
server_url: str,
|
|
api_key: str | None,
|
|
checksums: dict[str, str],
|
|
*,
|
|
timeout: float = CHECK_TIMEOUT_SECONDS,
|
|
) -> dict:
|
|
"""Ask Immich which of these exact bytes it already holds (US05-04).
|
|
|
|
``checksums`` maps an application key (the asset id) to the SHA-1 of the bytes
|
|
that were uploaded — the digest Immich itself deduplicates on. The answer is
|
|
``{"reachable", "detail", "present"}`` where ``present`` maps each key to
|
|
``True`` (the server rejected it as a duplicate, so it holds those bytes),
|
|
``False`` (the server would accept it, so it does not), or ``None`` (the server
|
|
answered something this adapter will not interpret).
|
|
|
|
An unreachable or unparsable server is reported, never guessed at: the caller
|
|
must treat it as uncertainty rather than absence.
|
|
"""
|
|
if not server_url or not api_key:
|
|
return {"reachable": False, "detail": "no Immich credentials configured", "present": {}}
|
|
keys = list(checksums)
|
|
present: dict[str, bool | None] = {}
|
|
for start in range(0, len(keys), CHECK_BATCH_SIZE):
|
|
# ponytail: fixed chunk size; make it configurable if a server ever rejects it.
|
|
chunk = keys[start : start + CHECK_BATCH_SIZE]
|
|
payload = {"assets": [{"id": key, "checksum": checksums[key]} for key in chunk]}
|
|
request = urllib.request.Request( # noqa: S310 — http(s) URL from configuration
|
|
server_url.rstrip("/") + BULK_CHECK_PATH,
|
|
data=json.dumps(payload).encode("utf-8"),
|
|
headers={"Content-Type": "application/json", "x-api-key": api_key},
|
|
method="POST",
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310
|
|
body = json.loads(response.read().decode("utf-8") or "{}")
|
|
except (urllib.error.URLError, OSError, ValueError, TimeoutError) as error:
|
|
return {"reachable": False, "detail": f"{type(error).__name__}: {error}", "present": {}}
|
|
results = body.get("results")
|
|
if not isinstance(results, list):
|
|
return {
|
|
"reachable": False,
|
|
"detail": "unrecognised bulk-upload-check response",
|
|
"present": {},
|
|
}
|
|
for result in results:
|
|
if not isinstance(result, dict) or result.get("id") not in checksums:
|
|
continue
|
|
present[result["id"]] = _holds_bytes(result)
|
|
return {"reachable": True, "detail": None, "present": present}
|
|
|
|
|
|
def _holds_bytes(result: dict) -> bool | None:
|
|
"""Whether one bulk-upload-check result means the server already has the file."""
|
|
action, reason = result.get("action"), result.get("reason")
|
|
if action == "reject":
|
|
# Only a duplicate proves possession; "unsupported-format" and friends say
|
|
# nothing about whether the bytes are there.
|
|
return True if reason == "duplicate" else None
|
|
return False if action == "accept" else None
|
|
|
|
|
|
def build_command(
|
|
*,
|
|
binary: str,
|
|
server_url: str,
|
|
api_key: str,
|
|
album_name: str,
|
|
folder: Path | str,
|
|
) -> list[str]:
|
|
"""The exact upload invocation for one folder-as-album batch."""
|
|
return [
|
|
binary,
|
|
"upload",
|
|
"from-folder",
|
|
f"--server={server_url}",
|
|
f"--api-key={api_key}",
|
|
f"--album-name={album_name}",
|
|
str(folder),
|
|
]
|
|
|
|
|
|
def redact(command: list[str]) -> list[str]:
|
|
"""The same command with every secret-bearing argument masked."""
|
|
return [f"--api-key={REDACTED}" if arg.startswith("--api-key=") else arg for arg in command]
|
|
|
|
|
|
def run_upload(
|
|
command: list[str],
|
|
*,
|
|
report_path: Path | str,
|
|
secret: str | None = None,
|
|
max_report_bytes: int | None = None, # resolved at call time; see MAX_REPORT_BYTES
|
|
cancelled: Callable[[], bool] | None = None,
|
|
timeout: float = UPLOAD_TIMEOUT_SECONDS,
|
|
) -> dict:
|
|
"""Run one upload and return its outcome.
|
|
|
|
``{"exit_code", "cancelled", "timed_out", "report_path", "report_bytes",
|
|
"report_truncated"}``. Output is streamed to ``report_path`` with ``secret``
|
|
masked and the file capped at ``max_report_bytes``; the pipe keeps being drained
|
|
after the cap so the child never blocks on a full buffer. ``cancelled`` is polled
|
|
while the process runs: when it returns true the uploader is asked to stop, then
|
|
killed if it does not.
|
|
"""
|
|
report_path = Path(report_path)
|
|
report_path.parent.mkdir(parents=True, exist_ok=True)
|
|
max_report_bytes = MAX_REPORT_BYTES if max_report_bytes is None else max_report_bytes
|
|
needle = (secret or "").encode() or None
|
|
written = 0
|
|
truncated = False
|
|
|
|
process = subprocess.Popen( # noqa: S603 — argv list, never a shell string
|
|
command,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
shell=False,
|
|
# Own process group: stopping the upload must stop whatever the uploader
|
|
# spawned too, not leave orphans holding the pipe open.
|
|
start_new_session=True,
|
|
)
|
|
|
|
def _drain() -> None:
|
|
nonlocal written, truncated
|
|
with open(report_path, "wb") as report:
|
|
for line in process.stdout: # line granularity keeps the mask reliable
|
|
if needle:
|
|
line = line.replace(needle, REDACTED.encode())
|
|
if written >= max_report_bytes:
|
|
truncated = True
|
|
continue # keep draining the pipe, stop growing the file
|
|
room = max_report_bytes - written
|
|
report.write(line[:room])
|
|
written += min(len(line), room)
|
|
truncated = truncated or len(line) > room
|
|
report.flush()
|
|
|
|
reader = threading.Thread(target=_drain, daemon=True)
|
|
reader.start()
|
|
|
|
stopped = timed_out = False
|
|
deadline = time.monotonic() + timeout
|
|
while process.poll() is None:
|
|
if cancelled is not None and cancelled():
|
|
stopped = True
|
|
elif time.monotonic() >= deadline:
|
|
timed_out = True
|
|
if stopped or timed_out:
|
|
_stop(process)
|
|
break
|
|
time.sleep(POLL_SECONDS)
|
|
|
|
exit_code = process.wait()
|
|
reader.join(timeout=DRAIN_SECONDS)
|
|
if process.stdout is not None:
|
|
process.stdout.close()
|
|
return {
|
|
"exit_code": exit_code,
|
|
"cancelled": stopped,
|
|
"timed_out": timed_out,
|
|
"report_path": str(report_path),
|
|
"report_bytes": written,
|
|
"report_truncated": truncated,
|
|
}
|
|
|
|
|
|
def _stop(process: subprocess.Popen) -> None:
|
|
"""Ask the uploader's whole process group to stop, then kill what remains."""
|
|
_signal_group(process, signal.SIGTERM)
|
|
try:
|
|
process.wait(timeout=TERMINATE_GRACE_SECONDS)
|
|
except subprocess.TimeoutExpired:
|
|
_signal_group(process, signal.SIGKILL)
|
|
process.wait()
|
|
|
|
|
|
def _signal_group(process: subprocess.Popen, sig: int) -> None:
|
|
try:
|
|
os.killpg(os.getpgid(process.pid), sig)
|
|
except (ProcessLookupError, PermissionError, OSError):
|
|
# No group (already reaped, or a platform without them): signal the child.
|
|
process.send_signal(sig)
|
|
|
|
|
|
def preview_command(
|
|
*, binary: str, server_url: str, album_name: str, folder: Path | str
|
|
) -> list[str]:
|
|
"""Redacted preview built without ever handling the real key."""
|
|
return redact(
|
|
build_command(
|
|
binary=binary,
|
|
server_url=server_url,
|
|
api_key=REDACTED,
|
|
album_name=album_name,
|
|
folder=folder,
|
|
)
|
|
)
|