224 lines
8.2 KiB
Python
224 lines
8.2 KiB
Python
"""Operational diagnostics: what the application is using, and what is about to
|
|
run out (US07-05, concept §17).
|
|
|
|
Every mutating stage in this application writes something before it is safe to
|
|
continue — a journal, an EXIF rewrite, an archive copy, a backup. All of them fail
|
|
badly on a full disk, so the sizes that grow (database, write-ahead log, thumbnail
|
|
cache, uploader reports, backups, logs) are reported separately rather than as one
|
|
opaque total, and each is compared against the free space actually left.
|
|
|
|
This is a read-only report. It never deletes, rotates, or prunes anything: what to
|
|
do about a warning is an operator's decision, and the tools for it are the
|
|
thumbnail cache quota, the backup retention helper, and log rotation outside the
|
|
application.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import functools
|
|
import json
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
from photo_pipeline.config import Config
|
|
from photo_pipeline.integrations import exiftool, immich_go
|
|
from photo_pipeline.services import app_lock
|
|
|
|
# Below this much free space, mutating stages should stop rather than risk a
|
|
# half-written journal, EXIF container, or archive copy.
|
|
LOW_DISK_BYTES = 1_000_000_000
|
|
CRITICAL_DISK_BYTES = 200_000_000
|
|
|
|
# Written into the container image at build time (US08-02). The image pins exiftool
|
|
# and immich-go, and this file is how a running container reports which versions it
|
|
# was built with — so a drifted or missing binary is visible here rather than in a
|
|
# failed EXIF checkpoint or a misparsed upload report.
|
|
IMAGE_VERSIONS_FILE = Path("/etc/photo-pipeline/versions.json")
|
|
|
|
|
|
def _tree_bytes(path: Path) -> int:
|
|
if not path.exists():
|
|
return 0
|
|
if path.is_file():
|
|
return path.stat().st_size
|
|
total = 0
|
|
for child in path.rglob("*"):
|
|
try:
|
|
if child.is_file() and not child.is_symlink():
|
|
total += child.stat().st_size
|
|
except OSError:
|
|
continue # vanished mid-walk; it is not using space any more
|
|
return total
|
|
|
|
|
|
def _component(name: str, path: Path, *, quota: int | None = None) -> dict:
|
|
used = _tree_bytes(path)
|
|
entry = {"name": name, "path": str(path), "bytes": used, "exists": path.exists()}
|
|
if quota is not None:
|
|
entry["quota_bytes"] = quota
|
|
entry["over_quota"] = used > quota
|
|
return entry
|
|
|
|
|
|
def disk(path: Path) -> dict:
|
|
"""Free/total for the filesystem holding ``path`` — the nearest existing parent,
|
|
so a data directory that does not exist yet still reports its future disk."""
|
|
probe = path
|
|
while not probe.exists() and probe != probe.parent:
|
|
probe = probe.parent
|
|
try:
|
|
usage = shutil.disk_usage(probe)
|
|
except OSError as error:
|
|
return {"path": str(probe), "error": str(error)}
|
|
return {
|
|
"path": str(probe),
|
|
"total_bytes": usage.total,
|
|
"free_bytes": usage.free,
|
|
"used_bytes": usage.used,
|
|
}
|
|
|
|
|
|
def _pinned_versions() -> dict[str, str]:
|
|
"""The versions this image recorded at build time; empty outside a container."""
|
|
try:
|
|
recorded = json.loads(IMAGE_VERSIONS_FILE.read_text())
|
|
except (OSError, ValueError):
|
|
return {}
|
|
if not isinstance(recorded, dict):
|
|
return {}
|
|
return {str(name): str(value) for name, value in recorded.items()}
|
|
|
|
|
|
@functools.lru_cache(maxsize=4)
|
|
def _uploader_version(binary: str) -> str | None:
|
|
"""Cached: the uploader cannot change version inside one process."""
|
|
return immich_go.version(binary)
|
|
|
|
|
|
def tools(config: Config) -> list[dict]:
|
|
"""The external executables the pipeline shells out to, and their versions.
|
|
|
|
``pinned`` is what the image was built against, ``version`` is what is actually
|
|
installed. They differ only when the binary was replaced or mounted over.
|
|
"""
|
|
return [
|
|
{
|
|
"name": "exiftool",
|
|
"path": exiftool.find_binary(),
|
|
"version": exiftool.version(),
|
|
"pinned": _pinned_versions().get("exiftool"),
|
|
},
|
|
{
|
|
"name": "immich-go",
|
|
"path": immich_go.find_binary(config.immich_go_binary),
|
|
"version": _uploader_version(config.immich_go_binary),
|
|
"pinned": _pinned_versions().get("immich-go"),
|
|
},
|
|
]
|
|
|
|
|
|
def report(config: Config) -> dict:
|
|
"""Sizes, disk headroom, tool versions, warnings, and who holds the library lock."""
|
|
database = config.database_path
|
|
components = [
|
|
_component("database", database),
|
|
_component("write_ahead_log", Path(f"{database}-wal")),
|
|
_component("shared_memory", Path(f"{database}-shm")),
|
|
_component(
|
|
"thumbnail_cache",
|
|
config.thumbnail_cache_dir,
|
|
quota=config.thumbnail_cache_quota_bytes,
|
|
),
|
|
_component("upload_reports", config.data_dir / "uploads"),
|
|
_component("backups", config.data_dir / "backups"),
|
|
_component("logs", config.data_dir / "logs"),
|
|
]
|
|
space = disk(config.data_dir)
|
|
free = space.get("free_bytes")
|
|
|
|
warnings: list[dict] = []
|
|
if free is not None and free < CRITICAL_DISK_BYTES:
|
|
warnings.append(
|
|
{
|
|
"code": "disk_critical",
|
|
"message": (
|
|
f"only {free} bytes free on {space['path']}; stop mutating stages "
|
|
"and free space before renaming, writing EXIF, or archiving"
|
|
),
|
|
}
|
|
)
|
|
elif free is not None and free < LOW_DISK_BYTES:
|
|
warnings.append(
|
|
{
|
|
"code": "disk_low",
|
|
"message": f"{free} bytes free on {space['path']}; prune backups or the cache",
|
|
}
|
|
)
|
|
for component in components:
|
|
if component.get("over_quota"):
|
|
warnings.append(
|
|
{
|
|
"code": "cache_over_quota",
|
|
"message": (
|
|
f"{component['name']} uses {component['bytes']} bytes, over its "
|
|
f"{component['quota_bytes']} byte quota"
|
|
),
|
|
}
|
|
)
|
|
# A write-ahead log that outgrows its database means checkpoints are starving —
|
|
# an operational warning, not something to ignore (concept §16).
|
|
wal = next(c for c in components if c["name"] == "write_ahead_log")
|
|
db = next(c for c in components if c["name"] == "database")
|
|
if wal["bytes"] > max(db["bytes"], 1) :
|
|
warnings.append(
|
|
{
|
|
"code": "wal_growth",
|
|
"message": (
|
|
f"the write-ahead log ({wal['bytes']} bytes) is larger than the database "
|
|
f"({db['bytes']} bytes); a long-running read may be blocking checkpoints"
|
|
),
|
|
}
|
|
)
|
|
|
|
installed_tools = tools(config)
|
|
for tool in installed_tools:
|
|
# A missing tool is reported as ``version: null`` rather than warned about: on a
|
|
# development machine the uploader is legitimately absent, and the stages that
|
|
# need it already refuse to run. A *drifted* tool is different — the image pinned
|
|
# a version and something replaced it.
|
|
if tool["version"] and tool["pinned"] and tool["pinned"] not in tool["version"]:
|
|
warnings.append(
|
|
{
|
|
"code": "tool_version_drift",
|
|
"message": (
|
|
f"{tool['name']} reports {tool['version']} but this image pinned "
|
|
f"{tool['pinned']}"
|
|
),
|
|
}
|
|
)
|
|
|
|
locks = {}
|
|
for role in ("api", "worker"):
|
|
holder = app_lock.LibraryLock(config, role).holder()
|
|
locks[role] = holder.as_dict() if holder else None
|
|
legacy = app_lock.legacy_activity(config)
|
|
if legacy["active"]:
|
|
warnings.append(
|
|
{
|
|
"code": "legacy_process_active",
|
|
"message": (
|
|
"a legacy CLI is writing this library; mutating stages are refused "
|
|
"until it stops"
|
|
),
|
|
}
|
|
)
|
|
return {
|
|
"components": components,
|
|
"total_bytes": sum(component["bytes"] for component in components),
|
|
"disk": space,
|
|
"tools": installed_tools,
|
|
"warnings": warnings,
|
|
"locks": locks,
|
|
"legacy_activity": legacy,
|
|
}
|