domverse/photoanalyzer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases Wiki Activity

US07-05: Deliver Backup and Operational Recovery (#92)

Browse Source
This commit was merged in pull request #92.
This commit is contained in:
domverse
2026-08-17 21:13:21 +02:00
parent d08d19c03c
commit 9851e112a9
19 changed files with 2057 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

221
photo_pipeline/services/app_lock.py Normal file
View File

@@ -0,0 +1,221 @@
"""Library-level process lock, and detection of an incompatible legacy run
(US07-05, concept §15 "migration and operational risks").
Every safety this application has — durable job leases, rename journals, archive
manifests — assumes that one installation owns the library. Two workers, or the
frozen CLI running beside the app, break that assumption *below* the level those
mechanisms can see: the second process simply does not know the first one's
database exists.
So mutation requires a file lock in the data directory, shaped as JSON so any
future or migrated entry point can read and honour it without importing this
package:
{"lock_version": 1, "role": "worker", "pid": 4242, "host": "...",
"started_at": "...", "library_roots": ["..."]}
One holder per role: an API and a worker are designed to run together, a second
worker is not. A lock whose process is gone is stale and is taken over with the
takeover recorded — refusing to start because of a crashed predecessor would turn
one outage into two.
Legacy detection is deliberately a heuristic, not a promise: the archived CLI has
no lock of its own, so what can be observed is its state files being written right
now. Recent writes to them mean something else is mutating this library, and every
mutating stage should refuse until it stops.
"""
from __future__ import annotations
import json
import os
import socket
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from photo_pipeline.config import Config
LOCK_VERSION = 1
LOCK_SUFFIX = ".lock.json"
# State files only the archived CLIs write. Their presence is history; a *recent*
# modification is a running process.
# ponytail: the real fix is a lock the migrated CLI paths take too — this catches
# the frozen archive, which has no lock and cannot be changed (US07-01).
LEGACY_ARTIFACTS = (
"photo_analyzer.db",
"nsfw_scores.csv",
"photo_analyzer_history.jsonl",
"photo_analyzer.log",
"photo_analyzer_debug.log",
)
LEGACY_ACTIVE_SECONDS = 300
class LockHeld(RuntimeError):
"""Another live process of the same role owns this library."""
def __init__(self, holder: "Holder") -> None:
super().__init__(
f"{holder.role} is already running for this library "
f"(pid {holder.pid} on {holder.host}, since {holder.started_at})"
)
self.holder = holder
class LegacyProcessActive(RuntimeError):
"""A legacy CLI appears to be mutating the same library right now."""
@dataclass(frozen=True)
class Holder:
role: str
pid: int
host: str
started_at: str
lock_version: int = LOCK_VERSION
library_roots: tuple[str, ...] = ()
@property
def alive(self) -> bool:
"""Whether the recorded process still exists on this host.
A lock from another host cannot be probed, so it is believed: assuming a
remote holder is dead is how two machines end up renaming the same folder.
"""
if self.host != socket.gethostname():
return True
try:
os.kill(self.pid, 0)
except ProcessLookupError:
return False
except PermissionError:
return True # exists, owned by someone else
return True
def as_dict(self) -> dict:
return {
"lock_version": self.lock_version,
"role": self.role,
"pid": self.pid,
"host": self.host,
"started_at": self.started_at,
"library_roots": list(self.library_roots),
"alive": self.alive,
}
def _now() -> datetime:
return datetime.now(timezone.utc)
def legacy_activity(config: Config) -> dict:
"""Legacy state files written within the activity window, if any."""
seen: list[dict] = []
cutoff = _now().timestamp() - LEGACY_ACTIVE_SECONDS
roots = [Path(root) for root in config.library_roots] + [Path(config.data_dir)]
for root in roots:
for name in LEGACY_ARTIFACTS:
path = root / name
try:
modified = path.stat().st_mtime
except OSError:
continue
if modified >= cutoff:
seen.append(
{
"path": str(path),
"modified_at": datetime.fromtimestamp(modified, timezone.utc).isoformat(),
}
)
return {"active": bool(seen), "artifacts": seen, "window_seconds": LEGACY_ACTIVE_SECONDS}
class LibraryLock:
"""One holder per role for one library. Used as a context manager."""
def __init__(self, config: Config, role: str = "worker") -> None:
self._config = config
self.role = role
self.path = Path(config.data_dir) / f"{role}{LOCK_SUFFIX}"
self._acquired = False
# ── inspection ────────────────────────────────────────────────────────────
def holder(self) -> Holder | None:
try:
payload = json.loads(self.path.read_text())
except (OSError, ValueError):
return None
try:
return Holder(
role=payload["role"],
pid=int(payload["pid"]),
host=payload["host"],
started_at=payload["started_at"],
lock_version=int(payload.get("lock_version", LOCK_VERSION)),
library_roots=tuple(payload.get("library_roots", ())),
)
except (KeyError, TypeError, ValueError):
# An unreadable lock is not an absent lock: something wrote it.
return Holder(role=self.role, pid=-1, host="unknown", started_at="unknown")
# ── acquire / release ─────────────────────────────────────────────────────
def acquire(self, *, allow_legacy: bool = False) -> Holder:
"""Take the lock for this role, or explain who has it.
Raises ``LockHeld`` when a live process of the same role owns the library,
and ``LegacyProcessActive`` when the archived CLI looks like it is running
against it.
"""
if not allow_legacy:
legacy = legacy_activity(self._config)
if legacy["active"]:
raise LegacyProcessActive(
"a legacy CLI is writing this library "
f"({', '.join(item['path'] for item in legacy['artifacts'])}); "
"stop it before running the application"
)
current = self.holder()
if current is not None:
if current.alive:
raise LockHeld(current)
# Stale: its process is gone. Take over, and say so.
self.path.unlink(missing_ok=True)
mine = Holder(
role=self.role,
pid=os.getpid(),
host=socket.gethostname(),
started_at=_now().isoformat(),
library_roots=tuple(str(root) for root in self._config.library_roots),
)
self.path.parent.mkdir(parents=True, exist_ok=True)
payload = {k: v for k, v in mine.as_dict().items() if k != "alive"}
# Exclusive create, so two processes racing here cannot both believe they won.
try:
with open(self.path, "x", encoding="utf-8") as handle:
json.dump(payload, handle, indent=2)
except FileExistsError:
winner = self.holder()
raise LockHeld(winner or mine) from None
self._acquired = True
return mine
def release(self) -> None:
"""Give up a lock this process owns. Another holder's lock is left alone."""
if not self._acquired:
return
current = self.holder()
if current is not None and current.pid == os.getpid():
self.path.unlink(missing_ok=True)
self._acquired = False
def __enter__(self) -> "LibraryLock":
self.acquire()
return self
def __exit__(self, *_) -> None:
self.release()

413
photo_pipeline/services/backup.py Normal file
View File

@@ -0,0 +1,413 @@
"""Online backups, verification, retention, and restore drills (US07-05).
A backup taken by copying a live SQLite file is not a backup: with WAL enabled the
file on disk is missing every committed page still in the write-ahead log, and a
writer mid-transaction makes the copy inconsistent. So every backup here goes
through SQLite's online backup API, which takes a consistent snapshot of a database
that is still being used (concept §3).
A backup directory holds exactly two things:
photo_pipeline.db the snapshot
manifest.json what it is, what it came from, and how to check it
The manifest is what makes the snapshot restorable by someone who was not there
when it was taken: the schema revision, the snapshot's SHA-256, the row counts it
should still have, the archive locations whose media the library depends on, and
which configuration values were set — **names and non-secret values only**. A
secret is recorded as "configured", never as its value, so a manifest can be
attached to a bug report.
Restore never writes into a live installation: it refuses a target that already
holds a database, because the one thing worse than a lost library is a half-merged
one. The drill is documented in README ("Backup and recovery").
"""
from __future__ import annotations
import hashlib
import json
import shutil
import sqlite3
from contextlib import closing
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from sqlalchemy import text
from photo_pipeline.config import Config
from photo_pipeline.db import create_db_engine, create_session_factory
SCHEMA_VERSION = 1
DB_NAME = "photo_pipeline.db"
MANIFEST_NAME = "manifest.json"
# How many backups the retention helper keeps by default. Small on purpose: a
# backup is a snapshot of state that is itself recoverable from the library, and
# the disk it lives on is the same one the low-disk warning watches.
DEFAULT_KEEP = 7
# Tables whose row counts are worth proving after a restore. Not the whole schema —
# these are the ones whose loss would be silent.
COUNTED_TABLES = (
"assets",
"asset_paths",
"safety_reviews",
"analysis_results",
"exif_projections",
"upload_batches",
"upload_items",
"archive_locations",
"archive_plans",
"archive_operations",
"rename_plans",
"rename_operations",
)
class BackupError(RuntimeError):
"""The backup could not be created, read, verified, or restored."""
@dataclass(frozen=True)
class VerifyResult:
ok: bool
issues: tuple[str, ...] = ()
revision: str | None = None
counts: dict | None = None
def as_dict(self) -> dict:
return {
"ok": self.ok,
"issues": list(self.issues),
"revision": self.revision,
"counts": self.counts,
}
def _now() -> datetime:
return datetime.now(timezone.utc)
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _revision(database: Path) -> str | None:
with closing(sqlite3.connect(database)) as connection:
try:
row = connection.execute("SELECT version_num FROM alembic_version").fetchone()
except sqlite3.Error:
return None
return row[0] if row else None
def _counts(database: Path) -> dict:
counts: dict[str, int] = {}
with closing(sqlite3.connect(database)) as connection:
for table in COUNTED_TABLES:
try:
counts[table] = connection.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
except sqlite3.Error:
continue # a table this revision does not have yet
return counts
def _integrity(database: Path) -> tuple[str, list[str]]:
"""``PRAGMA integrity_check`` plus ``foreign_key_check`` — structure and links.
Structural soundness is not referential soundness: a database can pass
``integrity_check`` and still hold an upload item pointing at an asset that
is gone.
"""
issues: list[str] = []
with closing(sqlite3.connect(database)) as connection:
try:
result = connection.execute("PRAGMA integrity_check").fetchone()[0]
if result != "ok":
issues.append(f"integrity_check: {result}")
violations = connection.execute("PRAGMA foreign_key_check").fetchall()
if violations:
issues.append(f"foreign_key_check: {len(violations)} violation(s)")
except sqlite3.DatabaseError as error:
issues.append(f"unreadable: {error}")
return "error", issues
return "ok" if not issues else "damaged", issues
def configuration_references(config: Config) -> dict:
"""Which configuration a restore has to reproduce — never the secrets themselves.
Paths and URLs are recorded because a restore into a fresh root has to be told
where the library and the Immich server were; API keys are recorded as
``configured`` so an operator knows one is required without the manifest ever
carrying it.
"""
return {
"data_dir": str(config.data_dir),
"database_path": str(config.database_path),
"library_roots": [str(root) for root in config.library_roots],
"thumbnail_cache_dir": str(config.thumbnail_cache_dir),
"immich_server_url": config.immich_server_url,
"immich_go_binary": config.immich_go_binary,
"secrets": {
"immich_api_key": "configured" if config.immich_api_key else "unset",
"vision_api_key": "configured" if config.vision_api_key else "unset",
},
}
def migrate_with_backup(config: Config) -> dict | None:
"""Upgrade the schema, with a snapshot first when there is state to lose.
A migration is the one routine operation that can damage every record at once,
and Alembic's own transaction does not cover SQLite DDL reliably. So a pending
upgrade is preceded by an online backup, and a failed upgrade names it in the
error: recovery is "restore that directory", not "reconstruct the library".
Returns the manifest of the backup it took, or ``None`` when none was needed.
"""
import logging
from photo_pipeline.db import run_migrations
service = BackupService(config)
manifest = service.pre_migration() if service.migration_pending() else None
try:
run_migrations(config.database_url)
except Exception:
if manifest is not None:
logging.getLogger(__name__).error(
"migration failed; restore the pre-migration backup at %s",
service.root / manifest["name"],
)
raise
return manifest
class BackupService:
def __init__(self, config: Config) -> None:
self._config = config
@property
def root(self) -> Path:
return self._config.data_dir / "backups"
# ── create ────────────────────────────────────────────────────────────────
def create(self, *, reason: str = "manual", keep: int | None = DEFAULT_KEEP) -> dict:
"""Take an online snapshot and describe it. Returns the manifest."""
source = self._config.database_path
if not source.exists():
raise BackupError(f"no database at {source}")
stamp = _now().strftime("%Y%m%dT%H%M%SZ")
safe_reason = "".join(c for c in reason if c.isalnum() or c in "-_") or "manual"
directory = self.root / f"{stamp}-{safe_reason}"
if directory.exists(): # same second, same reason
directory = self.root / f"{stamp}-{safe_reason}-{len(list(self.root.iterdir()))}"
directory.mkdir(parents=True)
target = directory / DB_NAME
try:
with closing(sqlite3.connect(source)) as src, closing(sqlite3.connect(target)) as dst:
src.backup(dst) # the online backup API, not a file copy
except (sqlite3.Error, OSError) as error:
shutil.rmtree(directory, ignore_errors=True)
raise BackupError(f"backup failed: {error}") from error
state, issues = _integrity(target)
manifest = {
"schema_version": SCHEMA_VERSION,
"name": directory.name,
"created_at": _now().isoformat(),
"reason": reason,
"revision": _revision(target),
"database": {
"name": DB_NAME,
"bytes": target.stat().st_size,
"sha256": sha256_file(target),
"integrity": state,
"issues": issues,
},
"counts": _counts(target),
"archive_locations": self._archive_locations(),
"configuration": configuration_references(self._config),
"retention": {
"keep": keep,
"guidance": (
"Keep the newest snapshot on a different disk than data_dir, and one "
"off-site copy per archive medium. A backup only covers the database: "
"the photos themselves live in the library and archive locations named "
"above, which need their own copies."
),
},
}
(directory / MANIFEST_NAME).write_text(json.dumps(manifest, indent=2))
if keep is not None:
manifest["pruned"] = self.prune(keep=keep)
return manifest
def migration_pending(self) -> bool:
"""True when the database exists and is not at the revision this code wants."""
from photo_pipeline.db import current_revision, head_revision
if not self._config.database_path.exists():
return False
return current_revision(self._config.database_url) != head_revision()
def pre_migration(self) -> dict | None:
"""Snapshot before a schema change, when there is something to lose.
Returns ``None`` when the database does not exist yet (a fresh install has
no state a failed migration could damage).
"""
if not self._config.database_path.exists():
return None
return self.create(reason="pre-migration")
def _archive_locations(self) -> list[dict]:
"""The media the library's archived originals live on.
A restored database still points at these; if they are not restored too,
the pictures are gone even though every record survived.
"""
engine = create_db_engine(self._config.database_url)
try:
factory = create_session_factory(engine)
with factory() as session:
rows = session.execute(
text("SELECT id, name, root, media_id, state FROM archive_locations")
).mappings().all()
except Exception:
return []
finally:
engine.dispose()
return [
{
"id": row["id"],
"name": row["name"],
"root": row["root"],
"media_id": row["media_id"],
"last_state": row["state"],
"mounted": Path(row["root"]).is_dir(),
}
for row in rows
]
# ── inspect ───────────────────────────────────────────────────────────────
def list(self) -> list[dict]:
"""Every backup, newest first, with what is known about it."""
if not self.root.is_dir():
return []
entries = []
for directory in sorted(self.root.iterdir(), reverse=True):
if not directory.is_dir():
continue
manifest = self.manifest(directory)
database = directory / DB_NAME
entries.append(
{
"name": directory.name,
"path": str(directory),
"created_at": (manifest or {}).get("created_at"),
"reason": (manifest or {}).get("reason"),
"revision": (manifest or {}).get("revision"),
"bytes": database.stat().st_size if database.exists() else 0,
"complete": bool(manifest) and database.exists(),
}
)
return entries
def manifest(self, directory: Path) -> dict | None:
path = Path(directory) / MANIFEST_NAME
if not path.exists():
return None
try:
return json.loads(path.read_text())
except ValueError:
return None
def verify(self, directory: Path | str) -> VerifyResult:
"""Prove a snapshot is still the one that was taken and still readable."""
directory = Path(directory)
if not directory.is_dir():
return VerifyResult(False, (f"no backup at {directory}",))
manifest = self.manifest(directory)
if manifest is None:
return VerifyResult(False, ("manifest is missing or unreadable",))
database = directory / manifest["database"]["name"]
if not database.exists():
return VerifyResult(False, ("the snapshot file is missing",), manifest.get("revision"))
issues: list[str] = []
if sha256_file(database) != manifest["database"]["sha256"]:
# Bit rot, a truncated copy, or an edited snapshot: all three mean the
# bytes are not the ones that were verified when the backup was made.
issues.append("sha256 does not match the manifest")
state, structural = _integrity(database)
issues.extend(structural)
counts = _counts(database) if state != "error" else None
if counts is not None and manifest.get("counts") and counts != manifest["counts"]:
issues.append(f"row counts changed: {manifest['counts']} -> {counts}")
return VerifyResult(not issues, tuple(issues), manifest.get("revision"), counts)
# ── retention ─────────────────────────────────────────────────────────────
def prune(self, *, keep: int = DEFAULT_KEEP) -> list[str]:
"""Delete the oldest backups beyond ``keep``. Never deletes the newest one."""
if keep < 1:
raise BackupError("retention must keep at least one backup")
removed = []
for entry in self.list()[keep:]:
shutil.rmtree(entry["path"], ignore_errors=True)
removed.append(entry["name"])
return removed
# ── restore ───────────────────────────────────────────────────────────────
def restore(self, directory: Path | str, target_data_dir: Path | str) -> dict:
"""Restore a verified snapshot into a **fresh** data directory.
Refuses a target that already holds a database. Restoring on top of a live
installation would merge two histories that disagree about which files were
renamed, uploaded, and archived — the one failure this whole story exists to
prevent. Recovering in place is: stop everything, move the old data
directory aside, restore into a new one.
"""
directory = Path(directory)
result = self.verify(directory)
if not result.ok:
raise BackupError(f"refusing to restore an unverified backup: {result.issues}")
target = Path(target_data_dir)
target.mkdir(parents=True, exist_ok=True)
destination = target / DB_NAME
if destination.exists():
raise BackupError(
f"{destination} already exists; restore into a fresh data directory"
)
shutil.copy2(directory / DB_NAME, destination)
# The write-ahead log of the *source* installation must not travel with a
# snapshot: the backup API already folded every committed page into it.
for leftover in (target / f"{DB_NAME}-wal", target / f"{DB_NAME}-shm"):
leftover.unlink(missing_ok=True)
restored = _integrity(destination)
return {
"backup": directory.name,
"restored_to": str(destination),
"revision": result.revision,
"counts": _counts(destination),
"integrity": restored[0],
"issues": restored[1],
"next_steps": [
"point PHOTO_PIPELINE_DATA_DIR at the restored directory",
"run `python -m photo_pipeline migrate` to reach the current revision",
"run an inventory scan so paths are reconciled against the real library",
"mount every archive location listed in the manifest before archiving again",
],
}

157
photo_pipeline/services/diagnostics.py Normal file
View File

@@ -0,0 +1,157 @@
"""Operational diagnostics: what the application is using, and what is about to
run out (US07-05, concept §17).
Every mutating stage in this application writes something before it is safe to
continue — a journal, an EXIF rewrite, an archive copy, a backup. All of them fail
badly on a full disk, so the sizes that grow (database, write-ahead log, thumbnail
cache, uploader reports, backups, logs) are reported separately rather than as one
opaque total, and each is compared against the free space actually left.
This is a read-only report. It never deletes, rotates, or prunes anything: what to
do about a warning is an operator's decision, and the tools for it are the
thumbnail cache quota, the backup retention helper, and log rotation outside the
application.
"""
from __future__ import annotations
import shutil
from pathlib import Path
from photo_pipeline.config import Config
from photo_pipeline.services import app_lock
# Below this much free space, mutating stages should stop rather than risk a
# half-written journal, EXIF container, or archive copy.
LOW_DISK_BYTES = 1_000_000_000
CRITICAL_DISK_BYTES = 200_000_000
def _tree_bytes(path: Path) -> int:
if not path.exists():
return 0
if path.is_file():
return path.stat().st_size
total = 0
for child in path.rglob("*"):
try:
if child.is_file() and not child.is_symlink():
total += child.stat().st_size
except OSError:
continue # vanished mid-walk; it is not using space any more
return total
def _component(name: str, path: Path, *, quota: int | None = None) -> dict:
used = _tree_bytes(path)
entry = {"name": name, "path": str(path), "bytes": used, "exists": path.exists()}
if quota is not None:
entry["quota_bytes"] = quota
entry["over_quota"] = used > quota
return entry
def disk(path: Path) -> dict:
"""Free/total for the filesystem holding ``path`` — the nearest existing parent,
so a data directory that does not exist yet still reports its future disk."""
probe = path
while not probe.exists() and probe != probe.parent:
probe = probe.parent
try:
usage = shutil.disk_usage(probe)
except OSError as error:
return {"path": str(probe), "error": str(error)}
return {
"path": str(probe),
"total_bytes": usage.total,
"free_bytes": usage.free,
"used_bytes": usage.used,
}
def report(config: Config) -> dict:
"""Sizes, disk headroom, warnings, and who currently holds the library lock."""
database = config.database_path
components = [
_component("database", database),
_component("write_ahead_log", Path(f"{database}-wal")),
_component("shared_memory", Path(f"{database}-shm")),
_component(
"thumbnail_cache",
config.thumbnail_cache_dir,
quota=config.thumbnail_cache_quota_bytes,
),
_component("upload_reports", config.data_dir / "uploads"),
_component("backups", config.data_dir / "backups"),
_component("logs", config.data_dir / "logs"),
]
space = disk(config.data_dir)
free = space.get("free_bytes")
warnings: list[dict] = []
if free is not None and free < CRITICAL_DISK_BYTES:
warnings.append(
{
"code": "disk_critical",
"message": (
f"only {free} bytes free on {space['path']}; stop mutating stages "
"and free space before renaming, writing EXIF, or archiving"
),
}
)
elif free is not None and free < LOW_DISK_BYTES:
warnings.append(
{
"code": "disk_low",
"message": f"{free} bytes free on {space['path']}; prune backups or the cache",
}
)
for component in components:
if component.get("over_quota"):
warnings.append(
{
"code": "cache_over_quota",
"message": (
f"{component['name']} uses {component['bytes']} bytes, over its "
f"{component['quota_bytes']} byte quota"
),
}
)
# A write-ahead log that outgrows its database means checkpoints are starving —
# an operational warning, not something to ignore (concept §16).
wal = next(c for c in components if c["name"] == "write_ahead_log")
db = next(c for c in components if c["name"] == "database")
if wal["bytes"] > max(db["bytes"], 1) :
warnings.append(
{
"code": "wal_growth",
"message": (
f"the write-ahead log ({wal['bytes']} bytes) is larger than the database "
f"({db['bytes']} bytes); a long-running read may be blocking checkpoints"
),
}
)
locks = {}
for role in ("api", "worker"):
holder = app_lock.LibraryLock(config, role).holder()
locks[role] = holder.as_dict() if holder else None
legacy = app_lock.legacy_activity(config)
if legacy["active"]:
warnings.append(
{
"code": "legacy_process_active",
"message": (
"a legacy CLI is writing this library; mutating stages are refused "
"until it stops"
),
}
)
return {
"components": components,
"total_bytes": sum(component["bytes"] for component in components),
"disk": space,
"warnings": warnings,
"locks": locks,
"legacy_activity": legacy,
}