"""immich-go report parsing (US05-03). The uploader's output is the only local evidence of what Immich did with each file, and its wording changes between releases (concept §15 "external integration risks"). So parsing is deliberately conservative: - **the parser is chosen by uploader version, not by guessing the format.** Each supported version family maps to one adapter with a pinned line grammar. An unrecognised version yields no adapter at all, which makes every item uncertain rather than optimistically successful. - **an unmatched line is never success.** Lines the adapter does not recognise are counted (``unparsed``) and kept as evidence; they never classify an item. - **evidence is bounded.** Each entry keeps a trimmed copy of the line that classified it, and the number of entries is capped, so a looping uploader cannot turn the report into unbounded database rows. Two grammars are supported. ``text-v1`` is the default human-readable log of the 0.21/0.22 line, ``json-v1`` the structured log (``--log-type=json``) of 0.23/0.24. Both classify through one shared phrase table, so the vocabulary cannot drift between them: ```text text-v1 INFO uploaded /lib/rome/a.jpg INFO server has the same file /lib/rome/b.jpg ERROR error uploading /lib/rome/e.jpg: connection reset Uploaded 1, duplicates 1, errors 1 json-v1 {"level":"INFO","msg":"uploaded","file":"/lib/rome/a.jpg"} {"level":"INFO","msg":"report","counts":{"uploaded":1}} ``` Nothing here touches the database; :mod:`photo_pipeline.services.upload_reports` turns a parse result into durable per-item outcomes. """ from __future__ import annotations import json import re PARSER_VERSION = 1 # Per-item outcomes. ``UNKNOWN`` is the safe default everywhere: it means the app # does not know what happened to that file and US05-04 must verify it. UPLOADED = "uploaded" UPGRADED = "upgraded" DUPLICATE = "duplicate" SKIPPED = "skipped" FAILED = "failed" UNKNOWN = "unknown" OUTCOMES = (UPLOADED, UPGRADED, DUPLICATE, SKIPPED, FAILED, UNKNOWN) # Longest/most specific phrases first: "server has an older file" also contains # "server has", and an error line about uploading also contains "upload". _PHRASES: tuple[tuple[str, str], ...] = ( ("server has the same file", DUPLICATE), ("server has an older file", UPGRADED), ("upgraded", UPGRADED), ("duplicate", DUPLICATE), ("discarded", SKIPPED), ("skipped", SKIPPED), ("error", FAILED), ("failed", FAILED), ("uploaded", UPLOADED), ("upload", UPLOADED), ) # Summary line of the text grammar: "Uploaded 3, duplicates 1, errors 2". _SUMMARY_WORDS = { "uploaded": UPLOADED, "upgraded": UPGRADED, "duplicates": DUPLICATE, "duplicate": DUPLICATE, "skipped": SKIPPED, "discarded": SKIPPED, "errors": FAILED, "error": FAILED, } _SUMMARY_PAIR = re.compile(r"([A-Za-z]+)\s+(\d+)") # A path is anything that looks absolute, up to an explanatory ": reason" tail. _PATH = re.compile(r"(/[^\s:][^:]*?)(?=:|$)") MAX_EVIDENCE_CHARS = 300 # The report file is already byte-capped; this caps the rows it can produce. MAX_ENTRIES = 20_000 def parser_for(uploader_version: str | None) -> str | None: """Adapter name for a recorded uploader version, or ``None`` when unsupported. Unsupported is not an error — it is the honest answer that this build's output format was never pinned, and it makes the whole report uncertain. """ match = re.search(r"(\d+)\.(\d+)", uploader_version or "") if match is None: return None return _SUPPORTED.get(f"{match.group(1)}.{match.group(2)}") def parse(report: str, uploader_version: str | None) -> dict: """Classify a raw report. Returns ``{"parser", "parser_version", "supported", "entries", "counts", "unparsed", "entries_truncated"}`` where ``entries`` is a list of ``{"path", "outcome", "evidence"}`` and ``counts`` is the uploader's own summary when it printed one (``None`` otherwise, never invented). """ parser = parser_for(uploader_version) result = { "parser": parser, "parser_version": PARSER_VERSION, "supported": parser is not None, "entries": [], "counts": None, "unparsed": 0, "entries_truncated": False, } if parser is None: return result read_line = _text_line if parser == "text-v1" else _json_line for raw in report.splitlines(): line = raw.strip() if not line: continue path, outcome, counts = read_line(line) if counts is not None: # Later summaries win: the uploader prints its totals once, at the end. result["counts"] = counts continue if path is None or outcome is None: result["unparsed"] += 1 continue if len(result["entries"]) >= MAX_ENTRIES: result["entries_truncated"] = True continue result["entries"].append( {"path": path, "outcome": outcome, "evidence": line[:MAX_EVIDENCE_CHARS]} ) return result def _classify(text: str) -> str | None: lowered = text.lower() for phrase, outcome in _PHRASES: if phrase in lowered: return outcome return None def _summary(text: str) -> dict[str, int] | None: """Totals from a summary line, or ``None`` when the line is not one.""" counts: dict[str, int] = {} for word, number in _SUMMARY_PAIR.findall(text): outcome = _SUMMARY_WORDS.get(word.lower()) if outcome is None: return None # an unknown noun means this is not the summary grammar counts[outcome] = counts.get(outcome, 0) + int(number) return counts or None def _text_line(line: str) -> tuple[str | None, str | None, dict | None]: path_match = _PATH.search(line) if path_match is None: return None, None, _summary(line) path = path_match.group(1).strip() # Classify from the words around the path, never from the path itself: an # album called "errors" must not turn an upload into a failure. context = line.replace(path, " ") return path, _classify(context), None def _json_line(line: str) -> tuple[str | None, str | None, dict | None]: try: record = json.loads(line) except ValueError: return None, None, None if not isinstance(record, dict): return None, None, None counts = record.get("counts") if isinstance(counts, dict): totals = { _SUMMARY_WORDS[key.lower()]: int(value) for key, value in counts.items() if key.lower() in _SUMMARY_WORDS and isinstance(value, int) } return None, None, totals or None path = record.get("file") message = record.get("msg") if not isinstance(path, str) or not isinstance(message, str): return None, None, None return path, _classify(message), None # Version family → adapter. Pinning this is the point: a build outside the list is # uncertain by construction (concept §15 "pin and record supported immich-go # versions"). _SUPPORTED = { "0.21": "text-v1", "0.22": "text-v1", "0.23": "json-v1", "0.24": "json-v1", } SUPPORTED_VERSIONS = tuple(sorted(_SUPPORTED))