US05-03: Parse and Persist Uploader Outcomes (#73)

photo_pipeline/services/upload_reports.py

@@ -0,0 +1,140 @@
+"""UploadReportService — durable per-item upload outcomes (US05-03).
+
+US05-02 proves the uploader ran and kept a bounded, credential-free report. This
+service is the step that reads that report and answers the operator's actual
+question — *what happened to each photo?* — without ever guessing in the app's
+favour:
+
+- **an item is ``unknown`` until the report says otherwise.** A file the report
+  never mentions, a report from an unpinned uploader version, and a line the
+  grammar does not recognise all leave the item ``unknown``. Exit code 0 is
+  evidence the *process* ended well, never evidence that a particular file reached
+  Immich (concept §15 "wrong upload state").
+- **counts are reconciled, not trusted.** The uploader's own summary is stored
+  next to the totals derived from the items. A disagreement makes the batch
+  ``requires_verification`` even when every line parsed.
+- **reprocessing is idempotent.** Outcomes are keyed by ``(batch_id, asset_id)``
+  and rewritten in place, so parsing the same report again — after a restart, or
+  because the operator asked — converges on the same rows and the same counts.
+
+The batch's own ``state`` still describes the process (US05-02's contract);
+``outcome_state`` describes the evidence. Turning ``requires_verification`` into
+verification, retry, and manual resolution is US05-04.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+from sqlalchemy import select
+from sqlalchemy.orm import sessionmaker
+
+from photo_pipeline.integrations import immich_go_report as report_parser
+from photo_pipeline.models import UploadBatch, UploadItem
+
+VERIFIED = "verified"
+REQUIRES_VERIFICATION = "requires_verification"
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _key(path: str) -> str:
+    return os.path.normpath(path)
+
+
+class UploadReportService:
+    def __init__(self, session_factory: sessionmaker) -> None:
+        self._session_factory = session_factory
+
+    def ingest(self, batch_id: str) -> dict:
+        """Parse this batch's report and persist an outcome for every item.
+
+        Returns ``{"parser", "parser_version", "outcome_state", "counts",
+        "report_counts", "unmatched", "unparsed"}``. Safe to call repeatedly.
+        """
+        with self._session_factory() as session:
+            batch = session.get(UploadBatch, batch_id)
+            if batch is None:
+                raise KeyError(f"unknown upload batch {batch_id!r}")
+            items = list(session.scalars(select(UploadItem).where(UploadItem.batch_id == batch_id)))
+            parsed = report_parser.parse(_read_report(batch.report_path), batch.uploader_version)
+
+            by_path = {_key(entry["path"]): entry for entry in parsed["entries"]}
+            # Basenames are the fallback: an uploader may log a path relative to the
+            # folder it was given. Ambiguous basenames are dropped rather than
+            # guessed at.
+            by_name: dict[str, dict] = {}
+            for key, entry in by_path.items():
+                name = os.path.basename(key)
+                by_name[name] = None if name in by_name else entry
+
+            matched: set[str] = set()
+            counts = {outcome: 0 for outcome in report_parser.OUTCOMES}
+            for item in items:
+                key = _key(item.path)
+                entry = by_path.get(key) or by_name.get(os.path.basename(key))
+                if entry is not None:
+                    matched.add(_key(entry["path"]))
+                item.outcome = entry["outcome"] if entry else report_parser.UNKNOWN
+                item.evidence = entry["evidence"] if entry else None
+                item.outcome_at = _now()
+                counts[item.outcome] += 1
+
+            unmatched = sorted(set(by_path) - matched)
+            reconciled = _reconciles(parsed["counts"], counts)
+            state = (
+                VERIFIED
+                if (
+                    parsed["supported"]
+                    and items
+                    and not counts[report_parser.UNKNOWN]
+                    and not unmatched
+                    and not parsed["entries_truncated"]
+                    and reconciled
+                )
+                else REQUIRES_VERIFICATION
+            )
+
+            batch.parser = parsed["parser"]
+            batch.parser_version = parsed["parser_version"]
+            batch.parsed_at = _now()
+            batch.outcome_state = state
+            batch.outcome_counts = json.dumps(counts, sort_keys=True)
+            batch.report_counts = (
+                json.dumps(parsed["counts"], sort_keys=True) if parsed["counts"] else None
+            )
+            session.commit()
+
+            return {
+                "parser": parsed["parser"],
+                "parser_version": parsed["parser_version"],
+                "outcome_state": state,
+                "counts": counts,
+                "report_counts": parsed["counts"],
+                "unmatched": unmatched,
+                "unparsed": parsed["unparsed"],
+            }
+
+
+def _read_report(report_path: str | None) -> str:
+    """The raw report, or ``""`` when the attempt never produced one."""
+    if not report_path:
+        return ""
+    path = Path(report_path)
+    return path.read_text(errors="replace") if path.exists() else ""
+
+
+def _reconciles(report_counts: dict | None, derived: dict) -> bool:
+    """Whether the uploader's own totals agree with the per-item outcomes.
+
+    No summary is not a disagreement — most of the uncertainty it would catch is
+    already caught by unmatched entries and unknown items.
+    """
+    if not report_counts:
+        return True
+    return all(derived.get(outcome) == total for outcome, total in report_counts.items())