US07-05: Deliver Backup and Operational Recovery (#92)
This commit was merged in pull request #92.
This commit is contained in:
221
photo_pipeline/services/app_lock.py
Normal file
221
photo_pipeline/services/app_lock.py
Normal file
@@ -0,0 +1,221 @@
|
||||
"""Library-level process lock, and detection of an incompatible legacy run
|
||||
(US07-05, concept §15 "migration and operational risks").
|
||||
|
||||
Every safety this application has — durable job leases, rename journals, archive
|
||||
manifests — assumes that one installation owns the library. Two workers, or the
|
||||
frozen CLI running beside the app, break that assumption *below* the level those
|
||||
mechanisms can see: the second process simply does not know the first one's
|
||||
database exists.
|
||||
|
||||
So mutation requires a file lock in the data directory, shaped as JSON so any
|
||||
future or migrated entry point can read and honour it without importing this
|
||||
package:
|
||||
|
||||
{"lock_version": 1, "role": "worker", "pid": 4242, "host": "...",
|
||||
"started_at": "...", "library_roots": ["..."]}
|
||||
|
||||
One holder per role: an API and a worker are designed to run together, a second
|
||||
worker is not. A lock whose process is gone is stale and is taken over with the
|
||||
takeover recorded — refusing to start because of a crashed predecessor would turn
|
||||
one outage into two.
|
||||
|
||||
Legacy detection is deliberately a heuristic, not a promise: the archived CLI has
|
||||
no lock of its own, so what can be observed is its state files being written right
|
||||
now. Recent writes to them mean something else is mutating this library, and every
|
||||
mutating stage should refuse until it stops.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from photo_pipeline.config import Config
|
||||
|
||||
LOCK_VERSION = 1
|
||||
LOCK_SUFFIX = ".lock.json"
|
||||
# State files only the archived CLIs write. Their presence is history; a *recent*
|
||||
# modification is a running process.
|
||||
# ponytail: the real fix is a lock the migrated CLI paths take too — this catches
|
||||
# the frozen archive, which has no lock and cannot be changed (US07-01).
|
||||
LEGACY_ARTIFACTS = (
|
||||
"photo_analyzer.db",
|
||||
"nsfw_scores.csv",
|
||||
"photo_analyzer_history.jsonl",
|
||||
"photo_analyzer.log",
|
||||
"photo_analyzer_debug.log",
|
||||
)
|
||||
LEGACY_ACTIVE_SECONDS = 300
|
||||
|
||||
|
||||
class LockHeld(RuntimeError):
|
||||
"""Another live process of the same role owns this library."""
|
||||
|
||||
def __init__(self, holder: "Holder") -> None:
|
||||
super().__init__(
|
||||
f"{holder.role} is already running for this library "
|
||||
f"(pid {holder.pid} on {holder.host}, since {holder.started_at})"
|
||||
)
|
||||
self.holder = holder
|
||||
|
||||
|
||||
class LegacyProcessActive(RuntimeError):
|
||||
"""A legacy CLI appears to be mutating the same library right now."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Holder:
|
||||
role: str
|
||||
pid: int
|
||||
host: str
|
||||
started_at: str
|
||||
lock_version: int = LOCK_VERSION
|
||||
library_roots: tuple[str, ...] = ()
|
||||
|
||||
@property
|
||||
def alive(self) -> bool:
|
||||
"""Whether the recorded process still exists on this host.
|
||||
|
||||
A lock from another host cannot be probed, so it is believed: assuming a
|
||||
remote holder is dead is how two machines end up renaming the same folder.
|
||||
"""
|
||||
if self.host != socket.gethostname():
|
||||
return True
|
||||
try:
|
||||
os.kill(self.pid, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return True # exists, owned by someone else
|
||||
return True
|
||||
|
||||
def as_dict(self) -> dict:
|
||||
return {
|
||||
"lock_version": self.lock_version,
|
||||
"role": self.role,
|
||||
"pid": self.pid,
|
||||
"host": self.host,
|
||||
"started_at": self.started_at,
|
||||
"library_roots": list(self.library_roots),
|
||||
"alive": self.alive,
|
||||
}
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def legacy_activity(config: Config) -> dict:
|
||||
"""Legacy state files written within the activity window, if any."""
|
||||
seen: list[dict] = []
|
||||
cutoff = _now().timestamp() - LEGACY_ACTIVE_SECONDS
|
||||
roots = [Path(root) for root in config.library_roots] + [Path(config.data_dir)]
|
||||
for root in roots:
|
||||
for name in LEGACY_ARTIFACTS:
|
||||
path = root / name
|
||||
try:
|
||||
modified = path.stat().st_mtime
|
||||
except OSError:
|
||||
continue
|
||||
if modified >= cutoff:
|
||||
seen.append(
|
||||
{
|
||||
"path": str(path),
|
||||
"modified_at": datetime.fromtimestamp(modified, timezone.utc).isoformat(),
|
||||
}
|
||||
)
|
||||
return {"active": bool(seen), "artifacts": seen, "window_seconds": LEGACY_ACTIVE_SECONDS}
|
||||
|
||||
|
||||
class LibraryLock:
|
||||
"""One holder per role for one library. Used as a context manager."""
|
||||
|
||||
def __init__(self, config: Config, role: str = "worker") -> None:
|
||||
self._config = config
|
||||
self.role = role
|
||||
self.path = Path(config.data_dir) / f"{role}{LOCK_SUFFIX}"
|
||||
self._acquired = False
|
||||
|
||||
# ── inspection ────────────────────────────────────────────────────────────
|
||||
|
||||
def holder(self) -> Holder | None:
|
||||
try:
|
||||
payload = json.loads(self.path.read_text())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
try:
|
||||
return Holder(
|
||||
role=payload["role"],
|
||||
pid=int(payload["pid"]),
|
||||
host=payload["host"],
|
||||
started_at=payload["started_at"],
|
||||
lock_version=int(payload.get("lock_version", LOCK_VERSION)),
|
||||
library_roots=tuple(payload.get("library_roots", ())),
|
||||
)
|
||||
except (KeyError, TypeError, ValueError):
|
||||
# An unreadable lock is not an absent lock: something wrote it.
|
||||
return Holder(role=self.role, pid=-1, host="unknown", started_at="unknown")
|
||||
|
||||
# ── acquire / release ─────────────────────────────────────────────────────
|
||||
|
||||
def acquire(self, *, allow_legacy: bool = False) -> Holder:
|
||||
"""Take the lock for this role, or explain who has it.
|
||||
|
||||
Raises ``LockHeld`` when a live process of the same role owns the library,
|
||||
and ``LegacyProcessActive`` when the archived CLI looks like it is running
|
||||
against it.
|
||||
"""
|
||||
if not allow_legacy:
|
||||
legacy = legacy_activity(self._config)
|
||||
if legacy["active"]:
|
||||
raise LegacyProcessActive(
|
||||
"a legacy CLI is writing this library "
|
||||
f"({', '.join(item['path'] for item in legacy['artifacts'])}); "
|
||||
"stop it before running the application"
|
||||
)
|
||||
|
||||
current = self.holder()
|
||||
if current is not None:
|
||||
if current.alive:
|
||||
raise LockHeld(current)
|
||||
# Stale: its process is gone. Take over, and say so.
|
||||
self.path.unlink(missing_ok=True)
|
||||
|
||||
mine = Holder(
|
||||
role=self.role,
|
||||
pid=os.getpid(),
|
||||
host=socket.gethostname(),
|
||||
started_at=_now().isoformat(),
|
||||
library_roots=tuple(str(root) for root in self._config.library_roots),
|
||||
)
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {k: v for k, v in mine.as_dict().items() if k != "alive"}
|
||||
# Exclusive create, so two processes racing here cannot both believe they won.
|
||||
try:
|
||||
with open(self.path, "x", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, indent=2)
|
||||
except FileExistsError:
|
||||
winner = self.holder()
|
||||
raise LockHeld(winner or mine) from None
|
||||
self._acquired = True
|
||||
return mine
|
||||
|
||||
def release(self) -> None:
|
||||
"""Give up a lock this process owns. Another holder's lock is left alone."""
|
||||
if not self._acquired:
|
||||
return
|
||||
current = self.holder()
|
||||
if current is not None and current.pid == os.getpid():
|
||||
self.path.unlink(missing_ok=True)
|
||||
self._acquired = False
|
||||
|
||||
def __enter__(self) -> "LibraryLock":
|
||||
self.acquire()
|
||||
return self
|
||||
|
||||
def __exit__(self, *_) -> None:
|
||||
self.release()
|
||||
Reference in New Issue
Block a user