US05-04: Verify, Retry, and Resolve Uncertain Uploads (#74)
This commit was merged in pull request #74.
This commit is contained in:
@@ -8,7 +8,9 @@ activity log. Both come from the same builder so the preview can never drift fro
|
||||
the command that would actually run.
|
||||
|
||||
Server reachability uses ``/api/server/ping`` through stdlib ``urllib`` — the app
|
||||
has no HTTP client dependency and this is one request.
|
||||
has no HTTP client dependency and this is one request. :func:`bulk_upload_check`
|
||||
uses the same client to ask Immich which uploaded bytes it already holds, which is
|
||||
the authoritative evidence behind upload verification (US05-04).
|
||||
|
||||
:func:`run_upload` is the only place the uploader is actually executed. It never
|
||||
uses a shell (the argument list goes straight to ``execve``, so no path or album
|
||||
@@ -35,6 +37,11 @@ from pathlib import Path
|
||||
REDACTED = "***"
|
||||
PING_PATH = "/api/server/ping"
|
||||
PING_TIMEOUT_SECONDS = 5.0
|
||||
# Immich's own deduplication endpoint: the authoritative answer to "do you already
|
||||
# have these exact bytes?" used to verify uncertain uploads (US05-04).
|
||||
BULK_CHECK_PATH = "/api/assets/bulk-upload-check"
|
||||
CHECK_TIMEOUT_SECONDS = 30.0
|
||||
CHECK_BATCH_SIZE = 500
|
||||
# Reports are kept in full up to this size; beyond it the tail is dropped and the
|
||||
# result is flagged truncated rather than growing without bound (concept §17).
|
||||
MAX_REPORT_BYTES = 4_000_000
|
||||
@@ -89,6 +96,68 @@ def ping(server_url: str, *, timeout: float = PING_TIMEOUT_SECONDS) -> tuple[boo
|
||||
return False, "server did not answer with pong"
|
||||
|
||||
|
||||
def bulk_upload_check(
|
||||
server_url: str,
|
||||
api_key: str | None,
|
||||
checksums: dict[str, str],
|
||||
*,
|
||||
timeout: float = CHECK_TIMEOUT_SECONDS,
|
||||
) -> dict:
|
||||
"""Ask Immich which of these exact bytes it already holds (US05-04).
|
||||
|
||||
``checksums`` maps an application key (the asset id) to the SHA-1 of the bytes
|
||||
that were uploaded — the digest Immich itself deduplicates on. The answer is
|
||||
``{"reachable", "detail", "present"}`` where ``present`` maps each key to
|
||||
``True`` (the server rejected it as a duplicate, so it holds those bytes),
|
||||
``False`` (the server would accept it, so it does not), or ``None`` (the server
|
||||
answered something this adapter will not interpret).
|
||||
|
||||
An unreachable or unparsable server is reported, never guessed at: the caller
|
||||
must treat it as uncertainty rather than absence.
|
||||
"""
|
||||
if not server_url or not api_key:
|
||||
return {"reachable": False, "detail": "no Immich credentials configured", "present": {}}
|
||||
keys = list(checksums)
|
||||
present: dict[str, bool | None] = {}
|
||||
for start in range(0, len(keys), CHECK_BATCH_SIZE):
|
||||
# ponytail: fixed chunk size; make it configurable if a server ever rejects it.
|
||||
chunk = keys[start : start + CHECK_BATCH_SIZE]
|
||||
payload = {"assets": [{"id": key, "checksum": checksums[key]} for key in chunk]}
|
||||
request = urllib.request.Request( # noqa: S310 — http(s) URL from configuration
|
||||
server_url.rstrip("/") + BULK_CHECK_PATH,
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json", "x-api-key": api_key},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310
|
||||
body = json.loads(response.read().decode("utf-8") or "{}")
|
||||
except (urllib.error.URLError, OSError, ValueError, TimeoutError) as error:
|
||||
return {"reachable": False, "detail": f"{type(error).__name__}: {error}", "present": {}}
|
||||
results = body.get("results")
|
||||
if not isinstance(results, list):
|
||||
return {
|
||||
"reachable": False,
|
||||
"detail": "unrecognised bulk-upload-check response",
|
||||
"present": {},
|
||||
}
|
||||
for result in results:
|
||||
if not isinstance(result, dict) or result.get("id") not in checksums:
|
||||
continue
|
||||
present[result["id"]] = _holds_bytes(result)
|
||||
return {"reachable": True, "detail": None, "present": present}
|
||||
|
||||
|
||||
def _holds_bytes(result: dict) -> bool | None:
|
||||
"""Whether one bulk-upload-check result means the server already has the file."""
|
||||
action, reason = result.get("action"), result.get("reason")
|
||||
if action == "reject":
|
||||
# Only a duplicate proves possession; "unsupported-format" and friends say
|
||||
# nothing about whether the bytes are there.
|
||||
return True if reason == "duplicate" else None
|
||||
return False if action == "accept" else None
|
||||
|
||||
|
||||
def build_command(
|
||||
*,
|
||||
binary: str,
|
||||
|
||||
Reference in New Issue
Block a user