US05-03: Parse and Persist Uploader Outcomes (#73)

photo_pipeline/integrations/immich_go_report.py

@@ -0,0 +1,204 @@
+"""immich-go report parsing (US05-03).
+
+The uploader's output is the only local evidence of what Immich did with each file,
+and its wording changes between releases (concept §15 "external integration
+risks"). So parsing is deliberately conservative:
+
+- **the parser is chosen by uploader version, not by guessing the format.** Each
+  supported version family maps to one adapter with a pinned line grammar. An
+  unrecognised version yields no adapter at all, which makes every item uncertain
+  rather than optimistically successful.
+- **an unmatched line is never success.** Lines the adapter does not recognise are
+  counted (``unparsed``) and kept as evidence; they never classify an item.
+- **evidence is bounded.** Each entry keeps a trimmed copy of the line that
+  classified it, and the number of entries is capped, so a looping uploader cannot
+  turn the report into unbounded database rows.
+
+Two grammars are supported. ``text-v1`` is the default human-readable log of the
+0.21/0.22 line, ``json-v1`` the structured log (``--log-type=json``) of 0.23/0.24.
+Both classify through one shared phrase table, so the vocabulary cannot drift
+between them:
+
+```text
+text-v1   INFO  uploaded /lib/rome/a.jpg
+          INFO  server has the same file /lib/rome/b.jpg
+          ERROR error uploading /lib/rome/e.jpg: connection reset
+          Uploaded 1, duplicates 1, errors 1
+
+json-v1   {"level":"INFO","msg":"uploaded","file":"/lib/rome/a.jpg"}
+          {"level":"INFO","msg":"report","counts":{"uploaded":1}}
+```
+
+Nothing here touches the database; :mod:`photo_pipeline.services.upload_reports`
+turns a parse result into durable per-item outcomes.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+
+PARSER_VERSION = 1
+
+# Per-item outcomes. ``UNKNOWN`` is the safe default everywhere: it means the app
+# does not know what happened to that file and US05-04 must verify it.
+UPLOADED = "uploaded"
+UPGRADED = "upgraded"
+DUPLICATE = "duplicate"
+SKIPPED = "skipped"
+FAILED = "failed"
+UNKNOWN = "unknown"
+
+OUTCOMES = (UPLOADED, UPGRADED, DUPLICATE, SKIPPED, FAILED, UNKNOWN)
+
+# Longest/most specific phrases first: "server has an older file" also contains
+# "server has", and an error line about uploading also contains "upload".
+_PHRASES: tuple[tuple[str, str], ...] = (
+    ("server has the same file", DUPLICATE),
+    ("server has an older file", UPGRADED),
+    ("upgraded", UPGRADED),
+    ("duplicate", DUPLICATE),
+    ("discarded", SKIPPED),
+    ("skipped", SKIPPED),
+    ("error", FAILED),
+    ("failed", FAILED),
+    ("uploaded", UPLOADED),
+    ("upload", UPLOADED),
+)
+
+# Summary line of the text grammar: "Uploaded 3, duplicates 1, errors 2".
+_SUMMARY_WORDS = {
+    "uploaded": UPLOADED,
+    "upgraded": UPGRADED,
+    "duplicates": DUPLICATE,
+    "duplicate": DUPLICATE,
+    "skipped": SKIPPED,
+    "discarded": SKIPPED,
+    "errors": FAILED,
+    "error": FAILED,
+}
+_SUMMARY_PAIR = re.compile(r"([A-Za-z]+)\s+(\d+)")
+# A path is anything that looks absolute, up to an explanatory ": reason" tail.
+_PATH = re.compile(r"(/[^\s:][^:]*?)(?=:|$)")
+
+MAX_EVIDENCE_CHARS = 300
+# The report file is already byte-capped; this caps the rows it can produce.
+MAX_ENTRIES = 20_000
+
+
+def parser_for(uploader_version: str | None) -> str | None:
+    """Adapter name for a recorded uploader version, or ``None`` when unsupported.
+
+    Unsupported is not an error — it is the honest answer that this build's output
+    format was never pinned, and it makes the whole report uncertain.
+    """
+    match = re.search(r"(\d+)\.(\d+)", uploader_version or "")
+    if match is None:
+        return None
+    return _SUPPORTED.get(f"{match.group(1)}.{match.group(2)}")
+
+
+def parse(report: str, uploader_version: str | None) -> dict:
+    """Classify a raw report.
+
+    Returns ``{"parser", "parser_version", "supported", "entries", "counts",
+    "unparsed", "entries_truncated"}`` where ``entries`` is a list of
+    ``{"path", "outcome", "evidence"}`` and ``counts`` is the uploader's own
+    summary when it printed one (``None`` otherwise, never invented).
+    """
+    parser = parser_for(uploader_version)
+    result = {
+        "parser": parser,
+        "parser_version": PARSER_VERSION,
+        "supported": parser is not None,
+        "entries": [],
+        "counts": None,
+        "unparsed": 0,
+        "entries_truncated": False,
+    }
+    if parser is None:
+        return result
+    read_line = _text_line if parser == "text-v1" else _json_line
+    for raw in report.splitlines():
+        line = raw.strip()
+        if not line:
+            continue
+        path, outcome, counts = read_line(line)
+        if counts is not None:
+            # Later summaries win: the uploader prints its totals once, at the end.
+            result["counts"] = counts
+            continue
+        if path is None or outcome is None:
+            result["unparsed"] += 1
+            continue
+        if len(result["entries"]) >= MAX_ENTRIES:
+            result["entries_truncated"] = True
+            continue
+        result["entries"].append(
+            {"path": path, "outcome": outcome, "evidence": line[:MAX_EVIDENCE_CHARS]}
+        )
+    return result
+
+
+def _classify(text: str) -> str | None:
+    lowered = text.lower()
+    for phrase, outcome in _PHRASES:
+        if phrase in lowered:
+            return outcome
+    return None
+
+
+def _summary(text: str) -> dict[str, int] | None:
+    """Totals from a summary line, or ``None`` when the line is not one."""
+    counts: dict[str, int] = {}
+    for word, number in _SUMMARY_PAIR.findall(text):
+        outcome = _SUMMARY_WORDS.get(word.lower())
+        if outcome is None:
+            return None  # an unknown noun means this is not the summary grammar
+        counts[outcome] = counts.get(outcome, 0) + int(number)
+    return counts or None
+
+
+def _text_line(line: str) -> tuple[str | None, str | None, dict | None]:
+    path_match = _PATH.search(line)
+    if path_match is None:
+        return None, None, _summary(line)
+    path = path_match.group(1).strip()
+    # Classify from the words around the path, never from the path itself: an
+    # album called "errors" must not turn an upload into a failure.
+    context = line.replace(path, " ")
+    return path, _classify(context), None
+
+
+def _json_line(line: str) -> tuple[str | None, str | None, dict | None]:
+    try:
+        record = json.loads(line)
+    except ValueError:
+        return None, None, None
+    if not isinstance(record, dict):
+        return None, None, None
+    counts = record.get("counts")
+    if isinstance(counts, dict):
+        totals = {
+            _SUMMARY_WORDS[key.lower()]: int(value)
+            for key, value in counts.items()
+            if key.lower() in _SUMMARY_WORDS and isinstance(value, int)
+        }
+        return None, None, totals or None
+    path = record.get("file")
+    message = record.get("msg")
+    if not isinstance(path, str) or not isinstance(message, str):
+        return None, None, None
+    return path, _classify(message), None
+
+
+# Version family → adapter. Pinning this is the point: a build outside the list is
+# uncertain by construction (concept §15 "pin and record supported immich-go
+# versions").
+_SUPPORTED = {
+    "0.21": "text-v1",
+    "0.22": "text-v1",
+    "0.23": "json-v1",
+    "0.24": "json-v1",
+}
+SUPPORTED_VERSIONS = tuple(sorted(_SUPPORTED))