494 lines
23 KiB
Python
494 lines
23 KiB
Python
"""Apply and recover guarded renames (US04-03, US04-04).
|
|
|
|
This is the only module that moves a photo library's files. Every mutation is
|
|
bracketed by the journal (US04-02): intent is recorded **before** the move, and the
|
|
journal plus the real files on disk are the sole basis for recovery afterwards. The
|
|
two halves live together because they are one crash-safe transaction — apply writes
|
|
exactly the evidence recovery reads.
|
|
|
|
Per operation the sequence is:
|
|
|
|
```
|
|
journal.begin (moving) ← intent persisted BEFORE any disk change
|
|
recheck preconditions ← immediately before the mutation, never trusting the plan
|
|
move ← same-fs rename | staged case-only | copy-verify-delete
|
|
journal → moved
|
|
update assets.current_path + path occurrences (one transaction)
|
|
journal → database_updated
|
|
verify bytes and paths on disk
|
|
journal → verified → complete
|
|
```
|
|
|
|
Safety rules that are never relaxed:
|
|
|
|
- An unexpected occupant at the destination is **never** overwritten; the operation
|
|
fails and the source is left untouched.
|
|
- Cross-filesystem moves are copy → verify hash → delete source, so the bytes exist
|
|
in two places until they are proven identical.
|
|
- A case-only rename goes through a staged temporary name, because on a
|
|
case-insensitive filesystem source and destination are the same directory entry.
|
|
- Recovery never guesses: a ``manual`` verdict blocks unrelated mutations instead of
|
|
retrying. Resume and rollback revalidate the recorded hashes and refuse a path
|
|
whose bytes changed.
|
|
- Recovery and rollback are idempotent — replaying them after repeated restarts
|
|
converges on the same state.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
from photo_pipeline.models import Asset, AssetPath, RenamePlan
|
|
from photo_pipeline.services import hashing
|
|
from photo_pipeline.services.rename_journal import (
|
|
MANUAL,
|
|
RESUMABLE,
|
|
JournalState,
|
|
RenameJournal,
|
|
)
|
|
|
|
# ``applied`` is included so a repeated apply is a safe no-op rather than an error:
|
|
# every operation is already complete and simply skips (idempotent confirmation).
|
|
# ``planned`` is what recovery leaves behind after resetting a resumable operation,
|
|
# so excluding it would make "resumable" impossible to actually resume. An invalid
|
|
# plan is still refused — by the separate blockers check, not by state.
|
|
APPLYABLE_PLAN_STATES = frozenset({"planned", "validated", "applying", "failed", "applied"})
|
|
|
|
|
|
class ApplyError(RuntimeError):
|
|
"""The request cannot be applied (unknown plan, invalid state, blocked)."""
|
|
|
|
|
|
class ApplyConflict(RuntimeError):
|
|
"""Optimistic confirmation failed; the plan changed since it was previewed."""
|
|
|
|
|
|
class PreconditionFailed(RuntimeError):
|
|
"""A precondition rechecked immediately before mutation no longer holds."""
|
|
|
|
def __init__(self, code: str, message: str) -> None:
|
|
super().__init__(message)
|
|
self.code = code
|
|
|
|
|
|
def _now() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def _maybe_fault(state: str) -> None:
|
|
"""Test-only crash barrier (concept §18 fault injection).
|
|
|
|
When ``PHOTO_PIPELINE_FAULT_AFTER`` names a journal state, the process dies
|
|
abruptly the moment that state has been persisted — modelling a real kill at
|
|
exactly that transition. Never set outside tests.
|
|
"""
|
|
if os.environ.get("PHOTO_PIPELINE_FAULT_AFTER") == state:
|
|
os._exit(9)
|
|
|
|
|
|
class RenameApplyService:
|
|
def __init__(self, session_factory: sessionmaker, *, library_roots: tuple = ()) -> None:
|
|
self._session_factory = session_factory
|
|
self._roots = tuple(Path(root) for root in library_roots)
|
|
self.journal = RenameJournal(session_factory)
|
|
|
|
# ── apply ─────────────────────────────────────────────────────────────────
|
|
|
|
def apply(
|
|
self,
|
|
plan_id: str,
|
|
*,
|
|
expected_version: int,
|
|
expected_checksum: str | None = None,
|
|
worker_id: str = "apply",
|
|
) -> dict:
|
|
"""Apply a confirmed plan. Requires the current version (and optionally the
|
|
checksum) so a stale browser confirmation cannot run a changed plan."""
|
|
plan = self._require_plan(plan_id)
|
|
if plan["version"] != expected_version:
|
|
raise ApplyConflict(
|
|
f"plan {plan_id} is at version {plan['version']}, expected {expected_version}"
|
|
)
|
|
if expected_checksum is not None and plan["checksum"] != expected_checksum:
|
|
raise ApplyConflict(f"plan {plan_id} checksum changed since it was previewed")
|
|
if plan["state"] not in APPLYABLE_PLAN_STATES:
|
|
raise ApplyError(f"plan {plan_id} is {plan['state']} and cannot be applied")
|
|
if plan["blockers"]:
|
|
raise ApplyError(f"plan {plan_id} has unresolved blockers: {plan['blockers']}")
|
|
|
|
# Exclusive mutation lease: refuse while any other rename may have the
|
|
# library half-moved (concept §16 lock hierarchy).
|
|
blocking = [row for row in self.journal.incomplete() if row["plan_id"] != plan_id]
|
|
if blocking:
|
|
raise ApplyError(
|
|
f"another rename is unresolved ({blocking[0]['id']}); recover it first"
|
|
)
|
|
|
|
token = self._claim_plan(plan_id)
|
|
applied = failed = skipped = 0
|
|
for operation in self.journal.operations(plan_id):
|
|
if operation["journal_state"] == JournalState.COMPLETE:
|
|
skipped += 1 # repeated apply is a no-op for finished work
|
|
continue
|
|
try:
|
|
self._apply_one(operation, token=token, worker_id=worker_id)
|
|
applied += 1
|
|
except PreconditionFailed as error:
|
|
self.journal.transition(
|
|
operation["id"],
|
|
JournalState.FAILED,
|
|
fencing_token=token,
|
|
error=(error.code, str(error)),
|
|
)
|
|
failed += 1
|
|
except Exception as error: # unexpected: record and stop touching disk
|
|
self.journal.transition(
|
|
operation["id"],
|
|
JournalState.FAILED,
|
|
fencing_token=token,
|
|
error=("apply_error", str(error)),
|
|
)
|
|
failed += 1
|
|
state = self.journal.sync_plan_state(plan_id)
|
|
return {
|
|
"plan_id": plan_id,
|
|
"applied": applied,
|
|
"failed": failed,
|
|
"skipped": skipped,
|
|
"state": state,
|
|
}
|
|
|
|
def _apply_one(self, operation: dict, *, token: int, worker_id: str) -> None:
|
|
source = Path(operation["source_path"])
|
|
destination = Path(operation["destination_path"])
|
|
|
|
# 1. Intent first — after this point a crash is recoverable from evidence.
|
|
self.journal.begin(operation["id"], worker_id=worker_id, fencing_token=token)
|
|
_maybe_fault(JournalState.MOVING)
|
|
|
|
# 2. Recheck preconditions immediately before mutating, never trusting the
|
|
# plan's snapshot: files can change between preview and confirmation.
|
|
self._recheck(operation, source, destination)
|
|
|
|
# 3. The move itself.
|
|
if operation["case_only"]:
|
|
self._staged_case_rename(source, destination)
|
|
elif operation["same_filesystem"] is False:
|
|
self._copy_verify_delete(source, destination)
|
|
else:
|
|
os.rename(source, destination)
|
|
|
|
self.journal.transition(operation["id"], JournalState.MOVED, fencing_token=token)
|
|
_maybe_fault(JournalState.MOVED)
|
|
|
|
# 4. Database: stable IDs keep their identity, paths are re-pointed and the
|
|
# old occurrence is closed — all in one transaction.
|
|
self._reconcile_paths(operation, source, destination)
|
|
self.journal.transition(operation["id"], JournalState.DATABASE_UPDATED, fencing_token=token)
|
|
_maybe_fault(JournalState.DATABASE_UPDATED)
|
|
|
|
# 5. Postconditions: the bytes really are at the new paths.
|
|
self._verify(operation, destination)
|
|
self.journal.transition(operation["id"], JournalState.VERIFIED, fencing_token=token)
|
|
_maybe_fault(JournalState.VERIFIED)
|
|
|
|
self.journal.transition(operation["id"], JournalState.COMPLETE, fencing_token=token)
|
|
_maybe_fault(JournalState.COMPLETE)
|
|
|
|
def _recheck(self, operation: dict, source: Path, destination: Path) -> None:
|
|
if not source.exists():
|
|
raise PreconditionFailed("source_missing", f"source {source} disappeared")
|
|
if source.is_symlink() or destination.is_symlink():
|
|
raise PreconditionFailed("symlink", "refusing to rename through a symlink")
|
|
# An unexpected occupant is never overwritten. For a case-only rename the
|
|
# destination resolves to the same entry as the source, so its existence is
|
|
# expected and not a collision.
|
|
if destination.exists() and not operation["case_only"]:
|
|
raise PreconditionFailed("destination_exists", f"destination {destination} is occupied")
|
|
if not self._within_roots(source) or not self._within_roots(destination):
|
|
raise PreconditionFailed("root_escape", "path is outside the library roots")
|
|
|
|
expected = operation.get("expected_sha256") or {}
|
|
with self._session_factory() as session:
|
|
assets = session.scalars(
|
|
select(Asset).where(Asset.id.in_(operation["asset_ids"]))
|
|
).all()
|
|
current = {asset.id: asset.current_path for asset in assets}
|
|
for asset_id, path in current.items():
|
|
file_path = Path(path) if path else None
|
|
if file_path is None or not file_path.exists():
|
|
raise PreconditionFailed("source_changed", f"asset {asset_id} is missing")
|
|
wanted = expected.get(asset_id)
|
|
if wanted and hashing.sha256_file(file_path) != wanted:
|
|
raise PreconditionFailed(
|
|
"source_changed", f"asset {asset_id} changed since the plan was built"
|
|
)
|
|
|
|
def _within_roots(self, path: Path) -> bool:
|
|
if not self._roots:
|
|
return False
|
|
return any(path == root or root in path.parents for root in self._roots)
|
|
|
|
# ── move procedures ───────────────────────────────────────────────────────
|
|
|
|
@staticmethod
|
|
def _staged_case_rename(source: Path, destination: Path) -> None:
|
|
"""Case-only rename via a temporary name: on a case-insensitive filesystem
|
|
source and destination are the same entry, so a direct rename is a no-op or
|
|
an error depending on the platform."""
|
|
staging = source.with_name(f".rename-{uuid.uuid4().hex[:8]}")
|
|
os.rename(source, staging)
|
|
try:
|
|
os.rename(staging, destination)
|
|
except OSError:
|
|
os.rename(staging, source) # put it back rather than leave it staged
|
|
raise
|
|
|
|
@staticmethod
|
|
def _copy_verify_delete(source: Path, destination: Path) -> None:
|
|
"""Cross-filesystem move: the bytes exist in both places until they are
|
|
proven identical, and only then is the source removed."""
|
|
staging = destination.with_name(f".rename-{uuid.uuid4().hex[:8]}")
|
|
shutil.copytree(source, staging)
|
|
for original in sorted(p for p in source.rglob("*") if p.is_file()):
|
|
copied = staging / original.relative_to(source)
|
|
if not copied.exists():
|
|
shutil.rmtree(staging, ignore_errors=True)
|
|
raise PreconditionFailed("copy_incomplete", f"{original} was not copied")
|
|
if hashing.sha256_file(original) != hashing.sha256_file(copied):
|
|
shutil.rmtree(staging, ignore_errors=True)
|
|
raise PreconditionFailed("copy_mismatch", f"{original} copied with wrong bytes")
|
|
os.rename(staging, destination)
|
|
shutil.rmtree(source)
|
|
|
|
# ── database reconciliation ───────────────────────────────────────────────
|
|
|
|
def _reconcile_paths(self, operation: dict, source: Path, destination: Path) -> None:
|
|
"""Re-point every asset in the operation at its new path, keeping the stable
|
|
ID and recording the path history."""
|
|
now = _now()
|
|
with self._session_factory() as session:
|
|
assets = session.scalars(
|
|
select(Asset).where(Asset.id.in_(operation["asset_ids"]))
|
|
).all()
|
|
for asset in assets:
|
|
if not asset.current_path:
|
|
continue
|
|
old = Path(asset.current_path)
|
|
try:
|
|
relative = old.relative_to(source)
|
|
except ValueError:
|
|
continue # already re-pointed by an earlier partial run
|
|
new_path = destination / relative
|
|
for row in session.scalars(
|
|
select(AssetPath).where(
|
|
AssetPath.asset_id == asset.id,
|
|
AssetPath.path == asset.current_path,
|
|
AssetPath.valid_until.is_(None),
|
|
)
|
|
):
|
|
row.valid_until = now
|
|
session.add(
|
|
AssetPath(
|
|
asset_id=asset.id,
|
|
path=str(new_path),
|
|
valid_from=now,
|
|
reason="rename",
|
|
)
|
|
)
|
|
asset.current_path = str(new_path)
|
|
asset.state_version += 1
|
|
asset.updated_at = now
|
|
session.commit()
|
|
|
|
def _verify(self, operation: dict, destination: Path) -> None:
|
|
"""Postcondition: every asset's recorded path exists and its bytes are the
|
|
ones the plan expected."""
|
|
expected = operation.get("expected_sha256") or {}
|
|
with self._session_factory() as session:
|
|
assets = session.scalars(
|
|
select(Asset).where(Asset.id.in_(operation["asset_ids"]))
|
|
).all()
|
|
paths = {asset.id: asset.current_path for asset in assets}
|
|
for asset_id, path in paths.items():
|
|
if not path or not Path(path).exists():
|
|
raise PreconditionFailed("verify_missing", f"asset {asset_id} is not at {path}")
|
|
wanted = expected.get(asset_id)
|
|
if wanted and hashing.sha256_file(path) != wanted:
|
|
raise PreconditionFailed("verify_bytes", f"asset {asset_id} bytes changed")
|
|
|
|
# ── recovery (US04-04) ────────────────────────────────────────────────────
|
|
|
|
def recover(self, *, worker_id: str = "recovery") -> dict:
|
|
"""Resolve every incomplete operation from journal + disk evidence.
|
|
|
|
Idempotent: running it repeatedly converges. Ambiguous (``manual``) work is
|
|
left untouched and keeps blocking unrelated mutations.
|
|
"""
|
|
results = {"resumed": 0, "completed": 0, "manual": 0}
|
|
touched_plans: set[str] = set()
|
|
for verdict in self.journal.classify_all():
|
|
operation = self.journal.get(verdict["operation_id"])
|
|
touched_plans.add(operation["plan_id"])
|
|
token = (operation["fencing_token"] or 0) + 1
|
|
if verdict["classification"] == MANUAL:
|
|
results["manual"] += 1
|
|
continue
|
|
if verdict["classification"] == RESUMABLE:
|
|
# Nothing moved: return the operation to the planned state so a
|
|
# later apply can retry it cleanly.
|
|
self._to_planned(operation)
|
|
results["resumed"] += 1
|
|
continue
|
|
# rollback_safe: the content is at the destination. Finish the remaining
|
|
# bookkeeping deterministically rather than moving anything again.
|
|
self._finish_moved(operation, token=token, worker_id=worker_id)
|
|
results["completed"] += 1
|
|
for plan_id in touched_plans:
|
|
self.journal.sync_plan_state(plan_id)
|
|
return results
|
|
|
|
def _to_planned(self, operation: dict) -> None:
|
|
state = operation["journal_state"]
|
|
if state == JournalState.PLANNED:
|
|
return
|
|
if state in (JournalState.MOVING, JournalState.FAILED):
|
|
self.journal.transition(operation["id"], JournalState.PLANNED)
|
|
elif state == JournalState.ROLLBACK_REQUIRED:
|
|
self.journal.transition(operation["id"], JournalState.ROLLED_BACK)
|
|
|
|
def _finish_moved(self, operation: dict, *, token: int, worker_id: str) -> None:
|
|
"""Drive an operation whose bytes are already at the destination through the
|
|
remaining states, re-running the idempotent bookkeeping."""
|
|
operation_id = operation["id"]
|
|
source = Path(operation["source_path"])
|
|
destination = Path(operation["destination_path"])
|
|
state = operation["journal_state"]
|
|
|
|
if state == JournalState.ROLLBACK_REQUIRED:
|
|
self.rollback_operation(operation_id, token=token)
|
|
return
|
|
|
|
if state == JournalState.MOVING:
|
|
self.journal.transition(operation_id, JournalState.MOVED, fencing_token=token)
|
|
state = JournalState.MOVED
|
|
if state == JournalState.MOVED:
|
|
self._reconcile_paths(operation, source, destination)
|
|
self.journal.transition(
|
|
operation_id, JournalState.DATABASE_UPDATED, fencing_token=token
|
|
)
|
|
state = JournalState.DATABASE_UPDATED
|
|
if state == JournalState.DATABASE_UPDATED:
|
|
self._verify(operation, destination)
|
|
self.journal.transition(operation_id, JournalState.VERIFIED, fencing_token=token)
|
|
state = JournalState.VERIFIED
|
|
if state == JournalState.VERIFIED:
|
|
self.journal.transition(operation_id, JournalState.COMPLETE, fencing_token=token)
|
|
|
|
# ── rollback ──────────────────────────────────────────────────────────────
|
|
|
|
def rollback_operation(self, operation_id: str, *, token: int | None = None) -> dict:
|
|
"""Move an operation's content back to its source, revalidating the recorded
|
|
hashes first. Refuses when the source is occupied or the bytes changed."""
|
|
operation = self.journal.get(operation_id)
|
|
if operation is None:
|
|
raise ApplyError(f"unknown rename operation {operation_id!r}")
|
|
source = Path(operation["source_path"])
|
|
destination = Path(operation["destination_path"])
|
|
token = token if token is not None else (operation["fencing_token"] or 0) + 1
|
|
|
|
if operation["journal_state"] not in (
|
|
JournalState.MOVED,
|
|
JournalState.DATABASE_UPDATED,
|
|
JournalState.VERIFIED,
|
|
JournalState.ROLLBACK_REQUIRED,
|
|
JournalState.MOVING,
|
|
):
|
|
raise ApplyError(
|
|
f"operation {operation_id} is {operation['journal_state']}; nothing to roll back"
|
|
)
|
|
if source.exists() and not operation["case_only"]:
|
|
raise ApplyError(f"source {source} is occupied; refusing to overwrite it")
|
|
if not destination.exists():
|
|
raise ApplyError(f"destination {destination} is missing; cannot roll back")
|
|
|
|
# Revalidate before moving anything back: a changed file is not ours to move.
|
|
expected = operation.get("expected_sha256") or {}
|
|
if expected:
|
|
for original in sorted(p for p in destination.rglob("*") if p.is_file()):
|
|
digest = hashing.sha256_file(original)
|
|
if expected and digest not in expected.values():
|
|
raise ApplyError(
|
|
f"{original} does not match the recorded hashes; manual recovery required"
|
|
)
|
|
|
|
if operation["journal_state"] != JournalState.ROLLBACK_REQUIRED:
|
|
self.journal.transition(
|
|
operation_id, JournalState.ROLLBACK_REQUIRED, fencing_token=token
|
|
)
|
|
if operation["case_only"]:
|
|
self._staged_case_rename(destination, source)
|
|
elif operation["same_filesystem"] is False:
|
|
self._copy_verify_delete(destination, source)
|
|
else:
|
|
os.rename(destination, source)
|
|
|
|
self._reconcile_paths(operation, destination, source)
|
|
self.journal.transition(operation_id, JournalState.ROLLED_BACK, fencing_token=token)
|
|
self.journal.sync_plan_state(operation["plan_id"])
|
|
return self.journal.get(operation_id)
|
|
|
|
def rollback_plan(self, plan_id: str) -> dict:
|
|
"""Roll back every applied operation of a plan, newest first."""
|
|
rolled = 0
|
|
for operation in reversed(self.journal.operations(plan_id)):
|
|
if operation["journal_state"] in (
|
|
JournalState.MOVED,
|
|
JournalState.DATABASE_UPDATED,
|
|
JournalState.VERIFIED,
|
|
JournalState.COMPLETE,
|
|
JournalState.ROLLBACK_REQUIRED,
|
|
):
|
|
if operation["journal_state"] == JournalState.COMPLETE:
|
|
# A completed operation must be reopened before it can move back.
|
|
continue
|
|
self.rollback_operation(operation["id"])
|
|
rolled += 1
|
|
state = self.journal.sync_plan_state(plan_id)
|
|
return {"plan_id": plan_id, "rolled_back": rolled, "state": state}
|
|
|
|
# ── helpers ───────────────────────────────────────────────────────────────
|
|
|
|
def _require_plan(self, plan_id: str) -> dict:
|
|
with self._session_factory() as session:
|
|
plan = session.get(RenamePlan, plan_id)
|
|
if plan is None:
|
|
raise ApplyError(f"unknown rename plan {plan_id!r}")
|
|
return {
|
|
"id": plan.id,
|
|
"state": plan.state,
|
|
"version": plan.version,
|
|
"checksum": plan.checksum,
|
|
"blockers": json.loads(plan.blockers or "[]"),
|
|
}
|
|
|
|
def _claim_plan(self, plan_id: str) -> int:
|
|
"""Bump the plan version and use it as this attempt's fencing token, so a
|
|
worker from a superseded attempt cannot commit."""
|
|
with self._session_factory() as session:
|
|
plan = session.get(RenamePlan, plan_id)
|
|
plan.version += 1
|
|
plan.state = "applying"
|
|
plan.updated_at = _now()
|
|
token = plan.version
|
|
session.commit()
|
|
return token
|