"""RestoreService — plan and execute safe restores (US06-04). Restore is archiving read backwards, with one decisive difference: it removes nothing. The archived copy stays on its medium, so every failure mode here costs at most a discarded temporary file. What restore must never do is *lose identity* — the asset that comes back is the same asset, with its duplicate decision, safety review, analysis, and upload history intact — or *overwrite* something in the active library. Preflight proves, per concept §9 "Restore": - the recorded medium is mounted and is the right one (marker ``media_id``); - every selected asset is archived, its archive copy exists, and it hashes to exactly the bytes the database recorded — a mismatch is ``divergent`` and is refused, never silently accepted as "the file"; - the destination lies inside the library, outside ``_IGNORE/``, and is free; a taken path is answered with a collision-free name, never an overwrite; - the library filesystem has room for the scope plus the configured reserve; - no rename, archive, or restore lease is holding the lane. Blocker codes: ``no_library_root``, ``location_offline``, ``wrong_volume``, ``unsafe_destination``, ``library_not_writable``, ``insufficient_capacity``, ``lock_conflict``, ``rename_pending``, ``archive_pending``, ``empty_scope``, ``not_archived``, ``archive_missing``, ``bytes_changed``. Per item the sequence is: ``` journal.begin (transferring) ← intent persisted BEFORE any disk change recheck: medium, hash, free destination, asset still archived copy to a temporary file beside the destination, fsync, hash it back atomically publish into the library journal → verified current_path = destination, availability = active, path occurrence opened journal → complete ``` Like archiving, the confirmation token is derived from the report, so a changed scope, a swapped medium, or a destination that filled up invalidates it. """ from __future__ import annotations import hashlib import json import os import shutil import uuid from datetime import datetime, timezone from pathlib import Path from sqlalchemy import select from sqlalchemy.orm import sessionmaker from photo_pipeline.config import Config from photo_pipeline.faults import maybe_fault from photo_pipeline.jobs.domain_handlers import ARCHIVE_LOCK, LIBRARY_WRITE_LOCK, UPLOAD_LOCK from photo_pipeline.models import ArchiveLocation, ArchiveOperation, ArchivePlan, Asset, AssetPath from photo_pipeline.path_policy import PathPolicyError, is_excluded, normalize_root, resolve_within from photo_pipeline.services import availability from photo_pipeline.services.archive_journal import ( MANUAL, RESTORE, RESUMABLE, ArchiveJournal, ArchiveState, ) from photo_pipeline.services.archive_transfer import ( _clean_temp_files, _fsync_dir, _plan_dict, copy_verify_publish, ) from photo_pipeline.services.archives import ArchiveError from photo_pipeline.services.hashing import sha256_file from photo_pipeline.services.jobs import JobService from photo_pipeline.services.rename_apply import PreconditionFailed from photo_pipeline.services.rename_journal import RenameJournal PREFLIGHT_VERSION = 1 TOKEN_PREFIX = f"r{PREFLIGHT_VERSION}" # What a restored file is called when its original name is taken. The suffix is # visible on purpose: a restore that quietly reuses a name is indistinguishable # from an overwrite. RESTORED_SUFFIX = "restored" LOCKS = (LIBRARY_WRITE_LOCK, UPLOAD_LOCK, ARCHIVE_LOCK) APPLYABLE_PLAN_STATES = frozenset({"planned", "applying", "failed", "complete"}) def _now() -> datetime: return datetime.now(timezone.utc) def _issue(code: str, message: str) -> dict: return {"code": code, "message": message} class RestoreService: def __init__(self, session_factory: sessionmaker, *, config: Config) -> None: self._session_factory = session_factory self._config = config self._roots = tuple(normalize_root(root) for root in config.library_roots) self.journal = ArchiveJournal(session_factory) # ── preflight ───────────────────────────────────────────────────────────── def preflight(self, location_id: str, asset_ids: list[str] | None = None) -> dict: """Validate a restore scope and issue its token. Nothing is written.""" with self._session_factory() as session: location = session.get(ArchiveLocation, location_id) if location is None: raise ArchiveError("unknown_location", f"unknown archive location {location_id!r}") root = Path(location.root) online = availability.location_online(location) marker = availability.read_marker(root) report = { "schema_version": PREFLIGHT_VERSION, "location": { "id": location.id, "name": location.name, "root": str(root), "media_id": location.media_id, "state": _location_state(root, marker, location.media_id), }, "blockers": [], } items = self._items(session, location, asset_ids, reachable=online) report["blockers"] += self._destination_blockers(report["location"]["state"], root) report["blockers"] += self._lock_blockers() report["items"] = items report["totals"] = { "assets": len(items), "blocked": sum(1 for item in items if item["blockers"]), "bytes": sum(item["byte_size"] or 0 for item in items), } report["capacity"] = self._capacity(report["totals"]["bytes"]) if not report["capacity"]["sufficient"]: report["blockers"].append( _issue( "insufficient_capacity", # No free-space number here: it drifts between two identical # preflights and the token is a digest of this text (US06-06). f"{report['totals']['bytes']} B plus a " f"{self._config.archive_free_space_reserve_bytes} B reserve do not fit in " "the library", ) ) if not items: report["blockers"].append( _issue("empty_scope", "no archived assets are in the selected scope") ) report["state"] = ( "ready" if not report["blockers"] and not report["totals"]["blocked"] else "blocked" ) report["token"] = _token(report) report["generated_at"] = _now().isoformat() return report def verify_token(self, token: str, location_id: str, asset_ids: list[str] | None = None) -> bool: return bool(token) and token == self.preflight(location_id, asset_ids)["token"] def _items( self, session, location: ArchiveLocation, asset_ids: list[str] | None, *, reachable: bool ) -> list[dict]: stmt = select(Asset).where(Asset.archive_location_id == location.id) if asset_ids is None: # A restored asset keeps its archive link; the default scope is only what # is still archived, so restoring twice is an empty scope, not a blocker. stmt = stmt.where(Asset.availability_state.in_(availability.ARCHIVED)) else: stmt = stmt.where(Asset.id.in_(asset_ids)) assets = list(session.scalars(stmt.order_by(Asset.archive_path))) if asset_ids is not None: unknown = sorted(set(asset_ids) - {asset.id for asset in assets}) if unknown: raise ArchiveError( "unknown_asset", f"not archived at this location: {', '.join(unknown)}" ) taken: set[str] = set() return [self._item(asset, location, reachable=reachable, taken=taken) for asset in assets] def _item(self, asset: Asset, location: ArchiveLocation, *, reachable: bool, taken: set) -> dict: source = Path(location.root) / (asset.archive_path or "") blockers: list[dict] = [] archive_sha256 = None if asset.availability_state not in availability.ARCHIVED: blockers.append( _issue("not_archived", f"asset {asset.id} is {asset.availability_state}") ) if reachable: if not source.exists(): blockers.append(_issue("archive_missing", f"{source} is not on the medium")) else: archive_sha256 = sha256_file(source) if asset.current_sha256 and archive_sha256 != asset.current_sha256: blockers.append( _issue( "bytes_changed", f"{source} holds bytes that are not the recorded ones; " "the archived copy is divergent", ) ) destination, destination_blockers = self._destination(asset, taken) blockers += destination_blockers if destination is not None: taken.add(str(destination)) return { "asset_id": asset.id, "archive_path": asset.archive_path, "source_path": str(source), "destination_path": str(destination) if destination else None, "expected_sha256": asset.current_sha256, "archive_sha256": archive_sha256, "byte_size": asset.byte_size, "availability_state": asset.availability_state, "blockers": blockers, } def _destination(self, asset: Asset, taken: set) -> tuple[Path | None, list[dict]]: """A free path inside the library that mirrors the archived layout. Restoring onto an existing file is never an option, so a taken name is answered with ``name (restored).ext`` — visible, ordinary, and impossible to confuse with an overwrite. """ if not self._roots: return None, [_issue("no_library_root", "no library root is configured")] root = self._roots[0] try: candidate = resolve_within(root, root / (asset.archive_path or "")) except PathPolicyError as error: return None, [_issue("unsafe_destination", str(error))] if is_excluded(candidate): return None, [ _issue("unsafe_destination", f"{candidate} is inside an excluded (_IGNORE/) tree") ] return _free_path(candidate, taken), [] def _destination_blockers(self, state: str, root: Path) -> list[dict]: blockers: list[dict] = [] if not self._roots: blockers.append(_issue("no_library_root", "no library root is configured")) elif not os.access(self._roots[0], os.W_OK): blockers.append( _issue("library_not_writable", f"{self._roots[0]} is not writable") ) if state == "offline": blockers.append( _issue("location_offline", f"the archive medium is not mounted at {root}") ) elif state == "wrong_volume": blockers.append(_issue("wrong_volume", f"{root} holds a different archive medium")) return blockers def _lock_blockers(self) -> list[dict]: blockers: list[dict] = [] jobs = JobService(self._session_factory) for lock in LOCKS: held = jobs.blockers(lock) if held: blockers.append( _issue("lock_conflict", f"the {lock} lane is busy: job {held[0]['id']}") ) if RenameJournal(self._session_factory).blocks_mutation(): blockers.append( _issue("rename_pending", "an unresolved rename must be recovered before restoring") ) if self.journal.blocks_mutation(): blockers.append( _issue( "archive_pending", "an unresolved archive or restore must be recovered before restoring", ) ) return blockers def _capacity(self, required: int) -> dict: reserve = self._config.archive_free_space_reserve_bytes free = shutil.disk_usage(self._roots[0]).free if self._roots else None return { "required_bytes": required, "reserve_bytes": reserve, "free_bytes": free, "sufficient": free is not None and free >= required + reserve, } # ── plans ───────────────────────────────────────────────────────────────── def create(self, location_id: str, asset_ids: list[str] | None = None, *, token: str) -> dict: preflight = self.preflight(location_id, asset_ids) if not token or token != preflight["token"]: raise ArchiveError("stale_token", "the restore preflight changed since it was approved") if preflight["state"] != "ready": codes = ", ".join(sorted({issue["code"] for issue in preflight["blockers"]})) or "-" blocked = sorted( {issue["code"] for item in preflight["items"] for issue in item["blockers"]} ) raise ArchiveError( "blocked", f"the restore scope is blocked: {', '.join(blocked) or codes}" ) plan_id = str(uuid.uuid4()) with self._session_factory() as session: session.add( ArchivePlan( id=plan_id, location_id=location_id, token=token, albums=json.dumps(asset_ids) if asset_ids is not None else None, direction=RESTORE, state="planned", schema_version=PREFLIGHT_VERSION, asset_count=preflight["totals"]["assets"], byte_size=preflight["totals"]["bytes"], ) ) session.flush() for sequence, item in enumerate(preflight["items"]): session.add( ArchiveOperation( id=str(uuid.uuid4()), plan_id=plan_id, direction=RESTORE, sequence=sequence, album=Path(item["archive_path"]).parent.name or "(root)", asset_id=item["asset_id"], source_path=item["source_path"], destination_path=item["destination_path"], archive_path=item["archive_path"], expected_sha256=item["expected_sha256"], byte_size=item["byte_size"], journal_state=ArchiveState.PLANNED, ) ) session.commit() return self.get(plan_id) def get(self, plan_id: str) -> dict | None: with self._session_factory() as session: plan = session.get(ArchivePlan, plan_id) if plan is None or plan.direction != RESTORE: return None report = _plan_dict(plan) report["operations"] = self.journal.operations(plan_id) return report def list(self) -> list[dict]: with self._session_factory() as session: rows = session.scalars( select(ArchivePlan) .where(ArchivePlan.direction == RESTORE) .order_by(ArchivePlan.created_at) ) return [_plan_dict(row) for row in rows] # ── apply ───────────────────────────────────────────────────────────────── def apply( self, plan_id: str, *, expected_version: int | None = None, worker_id: str = "restore" ) -> dict: plan = self._require_plan(plan_id) if expected_version is not None and plan["version"] != expected_version: raise ArchiveError( "stale_plan", f"plan {plan_id} is at version {plan['version']}, expected {expected_version}", ) if plan["state"] not in APPLYABLE_PLAN_STATES: raise ArchiveError("invalid_state", f"plan {plan_id} is {plan['state']}") blocking = [row for row in self.journal.incomplete() if row["plan_id"] != plan_id] if blocking: raise ArchiveError( "archive_pending", f"another archive operation is unresolved ({blocking[0]['id']}); recover it first", ) token = self._claim_plan(plan_id) location = self._location(plan["location_id"]) restored = failed = skipped = 0 for operation in self.journal.operations(plan_id): if operation["journal_state"] == ArchiveState.COMPLETE: skipped += 1 continue try: if operation["journal_state"] == ArchiveState.VERIFIED: self._finish(operation, token=token) else: self._restore_one(operation, location, token=token, worker_id=worker_id) restored += 1 except PreconditionFailed as error: self._fail(operation, token, error.code, str(error)) failed += 1 except Exception as error: # unexpected: record and stop touching disk self._fail(operation, token, "restore_error", str(error)) failed += 1 state = self.journal.sync_plan_state(plan_id) return { "plan_id": plan_id, "restored": restored, "failed": failed, "skipped": skipped, "state": state, } def _restore_one(self, operation: dict, location: dict, *, token: int, worker_id: str) -> None: source = Path(operation["source_path"]) destination = Path(operation["destination_path"]) # 1. Intent first; from here a crash is resolvable from journal + disk. self.journal.begin(operation["id"], worker_id=worker_id, fencing_token=token) maybe_fault(ArchiveState.TRANSFERRING) # 2. Recheck against the medium and the library as they are right now. self._recheck(operation, source, destination, location) destination.parent.mkdir(parents=True, exist_ok=True) # 3. Always copy: the archived original stays on its medium. copy_verify_publish(source, destination, operation["expected_sha256"]) _fsync_dir(destination.parent) if sha256_file(destination) != operation["expected_sha256"]: raise PreconditionFailed( "restore_mismatch", f"{destination} does not hold the expected bytes" ) self.journal.transition(operation["id"], ArchiveState.VERIFIED, fencing_token=token) maybe_fault(ArchiveState.VERIFIED) self._finish(self.journal.get(operation["id"]), token=token) def _finish(self, operation: dict, *, token: int) -> None: """Publish the restored file to the database. Idempotent, so recovery may replay it after a crash between the copy and the bookkeeping.""" destination = Path(operation["destination_path"]) if not destination.exists() or sha256_file(destination) != operation["expected_sha256"]: raise PreconditionFailed( "restore_unverified", f"{destination} is not a verified restored copy" ) self._record_restored(operation, destination) self.journal.transition(operation["id"], ArchiveState.COMPLETE, fencing_token=token) maybe_fault(ArchiveState.COMPLETE) def _recheck(self, operation: dict, source: Path, destination: Path, location: dict) -> None: root = Path(location["root"]) if not root.is_dir() or not (root / availability.MARKER_NAME).exists(): raise PreconditionFailed("location_offline", f"{root} is not the archive medium") if not source.exists(): raise PreconditionFailed("archive_missing", f"{source} is not on the medium") if source.is_symlink() or destination.is_symlink(): raise PreconditionFailed("symlink", "refusing to restore through a symlink") if destination.exists(): # Never overwrite: the plan's free path was taken since it was made. raise PreconditionFailed( "destination_exists", f"destination {destination} is occupied" ) if not self._inside_library(destination): raise PreconditionFailed( "destination_escape", f"{destination} is outside the library roots" ) if sha256_file(source) != operation["expected_sha256"]: self._mark_divergent(operation["asset_id"]) raise PreconditionFailed( "bytes_changed", f"{source} changed since the plan was approved" ) with self._session_factory() as session: asset = session.get(Asset, operation["asset_id"]) if asset is None or asset.availability_state not in availability.ARCHIVED: raise PreconditionFailed( "not_archived", f"asset {operation['asset_id']} is no longer archived" ) def _inside_library(self, destination: Path) -> bool: for root in self._roots: try: resolve_within(root, destination) return True except PathPolicyError: continue return False # ── database ────────────────────────────────────────────────────────────── def _record_restored(self, operation: dict, destination: Path) -> None: """The bytes are back in the library: open the new active occurrence and set availability. Identity, decisions, and history are untouched — that is the entire point of restoring rather than re-importing.""" now = _now() with self._session_factory() as session: asset = session.get(Asset, operation["asset_id"]) if asset is None: raise PreconditionFailed( "asset_missing", f"asset {operation['asset_id']} no longer exists" ) # A restored asset may be returning to a path it once held, so only an # *open* occurrence counts as already registered — that is what keeps # recovery idempotent without collapsing the path history. recorded = session.scalar( select(AssetPath).where( AssetPath.asset_id == asset.id, AssetPath.path == str(destination), AssetPath.valid_until.is_(None), ) ) if recorded is None: # idempotent: recovery may replay this session.add( AssetPath( asset_id=asset.id, path=str(destination), valid_from=now, reason="restore", ) ) asset.current_path = str(destination) asset.availability_state = availability.ACTIVE asset.missing_at = None # The archive copy stays where it is; keeping the link means a restored # asset still knows which medium holds its archived bytes. asset.archive_divergent_at = None asset.state_version += 1 asset.updated_at = now session.commit() def _mark_divergent(self, asset_id: str) -> None: """Record that the archived copy is not the recorded file. Durable, because the next restore attempt must not rediscover this from scratch.""" with self._session_factory() as session: asset = session.get(Asset, asset_id) if asset is None: return asset.archive_divergent_at = _now() asset.state_version += 1 session.commit() # ── recovery ────────────────────────────────────────────────────────────── def recover(self, *, worker_id: str = "restore-recovery") -> dict: """Resolve every incomplete restore from journal + disk evidence. A restore never removed anything, so ``resumable`` simply discards the temporary debris and re-plans the item; ``forward`` finishes the bookkeeping for a published file; ``manual`` is left untouched and keeps blocking. """ results = {"resumed": 0, "completed": 0, "manual": 0} touched: set[str] = set() for verdict in self.journal.classify_all(direction=RESTORE): operation = self.journal.get(verdict["operation_id"]) touched.add(operation["plan_id"]) token = (operation["fencing_token"] or 0) + 1 if verdict["classification"] == MANUAL: results["manual"] += 1 continue if verdict["classification"] == RESUMABLE: _clean_temp_files(Path(operation["destination_path"]).parent) self.journal.transition(operation["id"], ArchiveState.PLANNED, fencing_token=token) results["resumed"] += 1 continue try: self._finish(operation, token=token) results["completed"] += 1 except PreconditionFailed as error: self._fail(operation, token, error.code, str(error)) results["manual"] += 1 for plan_id in touched: self.journal.sync_plan_state(plan_id) return results def recovery_status(self) -> dict: verdicts = self.journal.classify_all(direction=RESTORE) return { "operations": verdicts, "manual": [v for v in verdicts if v["classification"] == MANUAL], "blocks_mutation": self.journal.blocks_mutation(), } # ── helpers ─────────────────────────────────────────────────────────────── def _fail(self, operation: dict, token: int, code: str, message: str) -> None: self.journal.transition( operation["id"], ArchiveState.FAILED, fencing_token=token, error=(code, message) ) def _require_plan(self, plan_id: str) -> dict: plan = self.get(plan_id) if plan is None: raise ArchiveError("unknown_plan", f"unknown restore plan {plan_id!r}") return plan def _location(self, location_id: str) -> dict: with self._session_factory() as session: location = session.get(ArchiveLocation, location_id) if location is None: raise ArchiveError("unknown_location", f"unknown archive location {location_id!r}") return {"id": location.id, "root": location.root, "media_id": location.media_id} def _claim_plan(self, plan_id: str) -> int: with self._session_factory() as session: plan = session.get(ArchivePlan, plan_id) plan.version += 1 plan.state = "applying" plan.updated_at = _now() token = plan.version session.commit() return token # ── module helpers ─────────────────────────────────────────────────────────── def _location_state(root: Path, marker: dict | None, media_id: str) -> str: if not root.is_dir() or marker is None: return "offline" return "online" if marker.get("media_id") == media_id else "wrong_volume" def _free_path(candidate: Path, taken: set) -> Path: """``a.jpg`` → ``a (restored).jpg`` → ``a (restored 2).jpg`` … ``taken`` holds the destinations already claimed by earlier items of the same plan, so two restores in one scope cannot plan the same path. """ if not candidate.exists() and str(candidate) not in taken: return candidate stem, suffix = candidate.stem, candidate.suffix attempt = 1 while True: label = RESTORED_SUFFIX if attempt == 1 else f"{RESTORED_SUFFIX} {attempt}" alternative = candidate.with_name(f"{stem} ({label}){suffix}") if not alternative.exists() and str(alternative) not in taken: return alternative attempt += 1 def _token(report: dict) -> str: """Digest of everything the report asserts about the scope and the medium. Free space is excluded: it drifts constantly without changing what a restore would do, and the capacity verdict itself is part of the digest. """ payload = {key: value for key, value in report.items() if key not in ("generated_at", "token")} payload["capacity"] = { key: value for key, value in payload["capacity"].items() if key != "free_bytes" } digest = hashlib.sha256( json.dumps(payload, sort_keys=True, ensure_ascii=False, default=str).encode("utf-8") ).hexdigest() return f"{TOKEN_PREFIX}:{digest}"