US05-04: Verify, Retry, and Resolve Uncertain Uploads (#74)

This commit was merged in pull request #74.
This commit is contained in:
2026-08-16 15:31:23 +02:00
parent 675876cad9
commit 3c440f3d43
10 changed files with 1186 additions and 17 deletions

View File

@@ -8,7 +8,9 @@ activity log. Both come from the same builder so the preview can never drift fro
the command that would actually run.
Server reachability uses ``/api/server/ping`` through stdlib ``urllib`` — the app
has no HTTP client dependency and this is one request.
has no HTTP client dependency and this is one request. :func:`bulk_upload_check`
uses the same client to ask Immich which uploaded bytes it already holds, which is
the authoritative evidence behind upload verification (US05-04).
:func:`run_upload` is the only place the uploader is actually executed. It never
uses a shell (the argument list goes straight to ``execve``, so no path or album
@@ -35,6 +37,11 @@ from pathlib import Path
REDACTED = "***"
PING_PATH = "/api/server/ping"
PING_TIMEOUT_SECONDS = 5.0
# Immich's own deduplication endpoint: the authoritative answer to "do you already
# have these exact bytes?" used to verify uncertain uploads (US05-04).
BULK_CHECK_PATH = "/api/assets/bulk-upload-check"
CHECK_TIMEOUT_SECONDS = 30.0
CHECK_BATCH_SIZE = 500
# Reports are kept in full up to this size; beyond it the tail is dropped and the
# result is flagged truncated rather than growing without bound (concept §17).
MAX_REPORT_BYTES = 4_000_000
@@ -89,6 +96,68 @@ def ping(server_url: str, *, timeout: float = PING_TIMEOUT_SECONDS) -> tuple[boo
return False, "server did not answer with pong"
def bulk_upload_check(
server_url: str,
api_key: str | None,
checksums: dict[str, str],
*,
timeout: float = CHECK_TIMEOUT_SECONDS,
) -> dict:
"""Ask Immich which of these exact bytes it already holds (US05-04).
``checksums`` maps an application key (the asset id) to the SHA-1 of the bytes
that were uploaded — the digest Immich itself deduplicates on. The answer is
``{"reachable", "detail", "present"}`` where ``present`` maps each key to
``True`` (the server rejected it as a duplicate, so it holds those bytes),
``False`` (the server would accept it, so it does not), or ``None`` (the server
answered something this adapter will not interpret).
An unreachable or unparsable server is reported, never guessed at: the caller
must treat it as uncertainty rather than absence.
"""
if not server_url or not api_key:
return {"reachable": False, "detail": "no Immich credentials configured", "present": {}}
keys = list(checksums)
present: dict[str, bool | None] = {}
for start in range(0, len(keys), CHECK_BATCH_SIZE):
# ponytail: fixed chunk size; make it configurable if a server ever rejects it.
chunk = keys[start : start + CHECK_BATCH_SIZE]
payload = {"assets": [{"id": key, "checksum": checksums[key]} for key in chunk]}
request = urllib.request.Request( # noqa: S310 — http(s) URL from configuration
server_url.rstrip("/") + BULK_CHECK_PATH,
data=json.dumps(payload).encode("utf-8"),
headers={"Content-Type": "application/json", "x-api-key": api_key},
method="POST",
)
try:
with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310
body = json.loads(response.read().decode("utf-8") or "{}")
except (urllib.error.URLError, OSError, ValueError, TimeoutError) as error:
return {"reachable": False, "detail": f"{type(error).__name__}: {error}", "present": {}}
results = body.get("results")
if not isinstance(results, list):
return {
"reachable": False,
"detail": "unrecognised bulk-upload-check response",
"present": {},
}
for result in results:
if not isinstance(result, dict) or result.get("id") not in checksums:
continue
present[result["id"]] = _holds_bytes(result)
return {"reachable": True, "detail": None, "present": present}
def _holds_bytes(result: dict) -> bool | None:
"""Whether one bulk-upload-check result means the server already has the file."""
action, reason = result.get("action"), result.get("reason")
if action == "reject":
# Only a duplicate proves possession; "unsupported-format" and friends say
# nothing about whether the bytes are there.
return True if reason == "duplicate" else None
return False if action == "accept" else None
def build_command(
*,
binary: str,