domverse/photoanalyzer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases Wiki Activity

US08-03: Compose the Runtime and Mount the Library Safely

Browse Source
This commit is contained in:
domverse
2026-08-19 19:46:20 +02:00
parent 888d859e93
commit 90df4be7f8
10 changed files with 1007 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
photo_pipeline/__main__.py
View File

@@ -11,9 +11,11 @@ from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Sequence
from photo_pipeline import path_policy
from photo_pipeline.config import Config
from photo_pipeline.services.app_lock import LegacyProcessActive, LibraryLock, LockHeld
from photo_pipeline.services.backup import BackupError, BackupService, migrate_with_backup
@@ -83,6 +85,18 @@ def main(argv: Sequence[str] | None = None) -> int:
config = Config.from_env()
config.database_path.parent.mkdir(parents=True, exist_ok=True)
# In a container the two roles that touch the library check their boundary before
# they take a lock or bind a port: the configured roots must name the mount paths,
# and a mismatch is cheaper to refuse than to discover at the first write (US08-03).
# Only in a container — on a host an unmounted root is an ordinary Tuesday (an
# archive medium that is not plugged in), and refusing to serve would take the
# offline half of the library away with it.
if args.command in ("serve", "worker") and path_policy.in_container():
refusal = path_policy.roots_refusal(config.library_roots, require_mount=True)
if refusal is not None:
print(refusal, file=sys.stderr)
return 5
if args.command == "migrate":
manifest = migrate_with_backup(config)
if manifest:
@@ -209,8 +223,6 @@ def main(argv: Sequence[str] | None = None) -> int:
lock.release()
return 0
import sys
import uvicorn
from photo_pipeline.api.app import ConfigurationRefused, create_app
@@ -238,8 +250,6 @@ def _acquire(lock: LibraryLock, *, allow_legacy: bool) -> int | None:
Returns an exit code to return, or ``None`` when the lock was acquired.
"""
import sys
try:
lock.acquire(allow_legacy=allow_legacy)
except LockHeld as error:

61
photo_pipeline/path_policy.py
View File

@@ -15,9 +15,7 @@ import os
from pathlib import Path
from typing import Iterable, Iterator
SUPPORTED_EXTENSIONS = {
".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif", ".tiff", ".tif"
}
SUPPORTED_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif", ".tiff", ".tif"}
EXCLUDED_DIR_NAMES = {"_IGNORE", ".@__thumb"}
@@ -72,6 +70,63 @@ def resolve_in_roots(roots: Iterable[os.PathLike | str], path: os.PathLike | str
raise PathPolicyError("path is outside the configured library roots")
def in_container() -> bool:
"""Whether this process is running inside a container image build of the app."""
return Path("/.dockerenv").exists()
def _under_mount(path: Path) -> bool:
"""Whether ``path`` or one of its parents below ``/`` is a mounted filesystem."""
current = path.resolve()
while current != current.parent:
if os.path.ismount(current):
return True
current = current.parent
return False
def roots_refusal(roots: Iterable[os.PathLike | str], *, require_mount: bool = False) -> str | None:
"""Why the configured library roots cannot be worked with, or ``None`` (US08-03).
The roots name the directories this installation renames folders in, rewrites
EXIF in, and archives from. Container path policy is the same problem as host
path policy with one new failure mode: the configured roots must name the
*container-side* mount paths. A host path configured inside a container is
either absent or an ordinary directory of the image, so the library looks empty
and the first write lands in the container's throwaway layer instead of in the
library. That is worth refusing at startup, while an operator is still watching,
rather than at the first write.
``require_mount`` is the container-only half of that: inside a container a real
library arrives through a bind mount, so a root that is not on (or under) a
mount point is not the library the deployment meant.
Writability is deliberately *not* checked: a bind mount's ownership is
virtualised by Docker Desktop and Colima (it arrives as ``root:root``), so
``os.access`` there is evidence about the virtio layer rather than about the
library. A wrong UID/GID surfaces as a refused rename with the real errno, which
is at least true; a refusal here would be false on two supported platforms.
"""
for root in roots:
path = Path(root)
if not path.exists():
return (
f"library root {path} does not exist: PHOTO_PIPELINE_LIBRARY_ROOTS must "
"name paths that exist here, and in a container that means the mount path"
)
if not path.is_dir():
return f"library root {path} is not a directory"
if not os.access(path, os.R_OK | os.X_OK):
return f"library root {path} is not readable by this process"
if require_mount and not _under_mount(path):
return (
f"library root {path} is not on a mounted filesystem in this container: "
"the library was not bind-mounted there, so PHOTO_PIPELINE_LIBRARY_ROOTS "
"names a directory of the image rather than the library"
)
return None
def iter_supported_files(root: os.PathLike | str) -> Iterator[Path]:
"""Yield supported, non-excluded files under ``root`` in deterministic order.

73
photo_pipeline/services/app_lock.py
View File

@@ -15,9 +15,17 @@ package:
"started_at": "...", "library_roots": ["..."]}
One holder per role: an API and a worker are designed to run together, a second
worker is not. A lock whose process is gone is stale and is taken over with the
takeover recorded — refusing to start because of a crashed predecessor would turn
one outage into two.
worker is not. A lock whose process is gone is stale and is taken over — refusing
to start because of a crashed predecessor would turn one outage into two.
Ownership is an advisory ``flock`` on that file, not the record inside it. The
record says *who*; the kernel says *whether*. That distinction is what makes the
lock work in containers (US08-03), where a PID and a hostname are namespaced: a
lock left behind by a container that no longer exists names a pid that still
"exists" in the new container and a host that cannot be probed, so believing the
file would deadlock every restart. A flock is released when its holder dies however
it dies, and is seen by every process that can open the file — which for a local
data directory is every container of this deployment.
Legacy detection is deliberately a heuristic, not a promise: the archived CLI has
no lock of its own, so what can be observed is its state files being written right
@@ -27,6 +35,7 @@ mutating stage should refuse until it stops.
from __future__ import annotations
import fcntl
import json
import os
import socket
@@ -109,6 +118,12 @@ def _now() -> datetime:
return datetime.now(timezone.utc)
def _in_container() -> bool:
from photo_pipeline import path_policy
return path_policy.in_container()
def legacy_activity(config: Config) -> dict:
"""Legacy state files written within the activity window, if any."""
seen: list[dict] = []
@@ -139,6 +154,7 @@ class LibraryLock:
self.role = role
self.path = Path(config.data_dir) / f"{role}{LOCK_SUFFIX}"
self._acquired = False
self._handle = None
# ── inspection ────────────────────────────────────────────────────────────
@@ -178,12 +194,31 @@ class LibraryLock:
"stop it before running the application"
)
self.path.parent.mkdir(parents=True, exist_ok=True)
# The kernel decides, because the file cannot: a container's PID and hostname
# are namespaced, so a lock left by a container that no longer exists names a
# pid that "exists" and a host that cannot be probed (US08-03). An advisory
# flock is held by a live process or by nobody, is released when that process
# dies however it dies, and is shared by every process that can open this
# file — which, for a local data directory, is every role in every container
# of this deployment.
handle = open(self.path, "a+", encoding="utf-8")
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
handle.close()
raise LockHeld(self.holder() or Holder(self.role, -1, "unknown", "unknown")) from None
current = self.holder()
if current is not None:
if current.alive:
raise LockHeld(current)
# Stale: its process is gone. Take over, and say so.
self.path.unlink(missing_ok=True)
if current is not None and current.host != socket.gethostname() and not _in_container():
# We hold the kernel's lock, so nothing on *this* machine holds the file.
# On a host that still leaves one case open: a data directory shared with
# another machine, whose flock we cannot trust. Believe its record rather
# than run two writers. In a container the data volume is local by
# construction (US08-03), and a foreign hostname is only a dead container.
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
handle.close()
raise LockHeld(current)
mine = Holder(
role=self.role,
@@ -192,15 +227,14 @@ class LibraryLock:
started_at=_now().isoformat(),
library_roots=tuple(str(root) for root in self._config.library_roots),
)
self.path.parent.mkdir(parents=True, exist_ok=True)
payload = {k: v for k, v in mine.as_dict().items() if k != "alive"}
# Exclusive create, so two processes racing here cannot both believe they won.
try:
with open(self.path, "x", encoding="utf-8") as handle:
json.dump(payload, handle, indent=2)
except FileExistsError:
winner = self.holder()
raise LockHeld(winner or mine) from None
handle.seek(0)
handle.truncate()
json.dump(payload, handle, indent=2)
handle.flush()
# Held open on purpose: closing it is what releases the lock, and that must
# happen when this process ends, not when this method returns.
self._handle = handle
self._acquired = True
return mine
@@ -208,9 +242,10 @@ class LibraryLock:
"""Give up a lock this process owns. Another holder's lock is left alone."""
if not self._acquired:
return
current = self.holder()
if current is not None and current.pid == os.getpid():
self.path.unlink(missing_ok=True)
self.path.unlink(missing_ok=True)
if self._handle is not None:
self._handle.close() # closing the descriptor releases the kernel lock
self._handle = None
self._acquired = False
def __enter__(self) -> "LibraryLock":