photoanalyzer/photo_pipeline/services/archive_journal.py

"""Archive transfer journal — the durable record of every per-file transition
(US06-02).

Archiving is the only stage that deletes an original, so the journal exists to make
one question answerable after any crash: *may this source file be removed?* Intent
is written before the mutation it describes, and the recorded state plus the real
files on disk are the sole basis for answering it later. This module owns the state
machine, the durable writes, and the evidence table; it never touches a photo
(:mod:`photo_pipeline.services.archive_transfer` does).

Per-item state machine (concept §9 "Transfer and removal semantics"):

```
planned → transferring → verified → removing → complete
              ↘             ↘          ↘ failed
```

A restore (US06-04) uses the same rows with ``direction='restore'``: it copies from
the medium back into the library and removes nothing, so it goes ``verified →
complete`` directly. ``source_path``/``destination_path`` always mean "from"/"to",
which is why the evidence table below needs no direction of its own.

- ``transferring`` — intent recorded; a temporary copy may exist, the destination
  may or may not have been published. Nothing has been removed.
- ``verified``     — the archived bytes exist at their final path, hash exactly as
  recorded, and the manifest entry is durable. Only from here may a source go.
- ``removing``     — the source removal is committed to; the source may already be
  gone while the database still points at it.
- ``complete``     — source absent, database updated, availability recorded.

``classify`` labels each incomplete item from the journal plus disk evidence:

- ``resumable`` — nothing was published; the source is intact, so applying again is
  safe.
- ``forward``   — the archived copy exists and matches its recorded hash, so the
  remaining steps (manifest, removal, bookkeeping) can be finished deterministically.
- ``manual``    — the evidence contradicts the journal (missing archive copy, wrong
  bytes, source and archive both gone). Nothing is guessed and nothing is removed;
  the item blocks unrelated mutations until a human decides.
"""

from __future__ import annotations

from datetime import datetime, timezone
from pathlib import Path

from sqlalchemy import select
from sqlalchemy.orm import sessionmaker

from photo_pipeline.models import ArchiveOperation, ArchivePlan
from photo_pipeline.services.hashing import sha256_file


class ArchiveState:
    PLANNED = "planned"
    TRANSFERRING = "transferring"
    VERIFIED = "verified"
    REMOVING = "removing"
    COMPLETE = "complete"
    FAILED = "failed"


ALLOWED_TRANSITIONS = {
    ArchiveState.PLANNED: {ArchiveState.TRANSFERRING, ArchiveState.FAILED},
    # From `transferring` the outcome is unknown until evidence is gathered, so it
    # may resolve forward, back to planned (proven nothing was published), or fail.
    ArchiveState.TRANSFERRING: {
        ArchiveState.VERIFIED,
        ArchiveState.PLANNED,
        ArchiveState.FAILED,
    },
    ArchiveState.VERIFIED: {ArchiveState.REMOVING, ArchiveState.FAILED},
    # No path back: once the source may be gone, only finishing is safe.
    ArchiveState.REMOVING: {ArchiveState.COMPLETE, ArchiveState.FAILED},
    ArchiveState.COMPLETE: set(),
    # A retry re-enters `transferring`, which rechecks every precondition from
    # scratch; recovery may also reset a failed item to `planned`.
    ArchiveState.FAILED: {ArchiveState.PLANNED, ArchiveState.TRANSFERRING},
}

# A restore removes nothing, so it has no ``removing`` step: a verified published
# copy is the whole job (US06-04). Keeping this as a separate table means the
# archive direction still cannot reach ``complete`` without going through removal.
RESTORE_TRANSITIONS = {
    **ALLOWED_TRANSITIONS,
    ArchiveState.VERIFIED: {ArchiveState.COMPLETE, ArchiveState.FAILED},
}

TERMINAL_STATES = frozenset({ArchiveState.COMPLETE})
# States where this item may already have touched the filesystem.
UNSAFE_STATES = frozenset({ArchiveState.TRANSFERRING, ArchiveState.VERIFIED, ArchiveState.REMOVING})

# Which way the bytes move. Same rows, same evidence table, opposite direction.
ARCHIVE = "archive"
RESTORE = "restore"

RESUMABLE = "resumable"
FORWARD = "forward"
MANUAL = "manual"


class JournalError(RuntimeError):
    pass


class InvalidTransition(JournalError):
    pass


class JournalConflict(JournalError):
    """Fencing check failed; a newer owner has taken over this operation."""


def can_transition(current: str, target: str, direction: str = ARCHIVE) -> bool:
    table = RESTORE_TRANSITIONS if direction == RESTORE else ALLOWED_TRANSITIONS
    return target in table.get(current, set())


def _now() -> datetime:
    return datetime.now(timezone.utc)


class ArchiveJournal:
    def __init__(self, session_factory: sessionmaker) -> None:
        self._session_factory = session_factory

    # ── intent ────────────────────────────────────────────────────────────────

    def begin(self, operation_id: str, *, worker_id: str, fencing_token: int) -> dict:
        """Record the intent to transfer **before** touching the filesystem."""
        with self._session_factory() as session:
            row = self._require(session, operation_id)
            if row.fencing_token is not None and fencing_token < row.fencing_token:
                raise JournalConflict(
                    f"stale fencing token {fencing_token} (current {row.fencing_token})"
                )
            if row.journal_state in TERMINAL_STATES:
                raise InvalidTransition(f"{row.journal_state} is terminal")
            if row.journal_state != ArchiveState.TRANSFERRING and not can_transition(
                row.journal_state, ArchiveState.TRANSFERRING, row.direction
            ):
                raise InvalidTransition(f"{row.journal_state} -> {ArchiveState.TRANSFERRING}")
            if row.journal_state != ArchiveState.TRANSFERRING:
                row.attempt_count += 1
            row.journal_state = ArchiveState.TRANSFERRING
            row.worker_id = worker_id
            row.fencing_token = fencing_token
            row.error_code = row.error_message = None
            row.updated_at = _now()
            session.commit()
            return _operation_dict(row)

    # ── transitions ───────────────────────────────────────────────────────────

    def transition(
        self,
        operation_id: str,
        target: str,
        *,
        fencing_token: int | None = None,
        error: tuple[str, str] | None = None,
        same_filesystem: bool | None = None,
    ) -> dict:
        """Move one operation to ``target``, enforcing the state machine.

        Re-entering the state an operation already holds is a no-op, which is what
        makes recovery idempotent across repeated restarts.
        """
        with self._session_factory() as session:
            row = self._require(session, operation_id)
            if fencing_token is not None and row.fencing_token is not None:
                if fencing_token < row.fencing_token:
                    raise JournalConflict(
                        f"stale fencing token {fencing_token} (current {row.fencing_token})"
                    )
            if same_filesystem is not None:
                row.same_filesystem = same_filesystem
            if row.journal_state == target:
                session.commit()
                return _operation_dict(row)  # idempotent
            if not can_transition(row.journal_state, target, row.direction):
                raise InvalidTransition(f"{row.journal_state} -> {target}")

            row.journal_state = target
            row.updated_at = _now()
            if target == ArchiveState.VERIFIED:
                row.verified_at = _now()
            if target == ArchiveState.COMPLETE:
                row.removed_at = _now()
            if error:
                row.error_code, row.error_message = error[0], error[1][:500]
            elif target != ArchiveState.FAILED:
                row.error_code = row.error_message = None
            session.commit()
            return _operation_dict(row)

    # ── reads ─────────────────────────────────────────────────────────────────

    def get(self, operation_id: str) -> dict | None:
        with self._session_factory() as session:
            row = session.get(ArchiveOperation, operation_id)
            return _operation_dict(row) if row else None

    def operations(self, plan_id: str) -> list[dict]:
        with self._session_factory() as session:
            rows = session.scalars(
                select(ArchiveOperation)
                .where(ArchiveOperation.plan_id == plan_id)
                .order_by(ArchiveOperation.sequence)
            )
            return [_operation_dict(row) for row in rows]

    def incomplete(self, *, direction: str | None = None) -> list[dict]:
        """Every operation left in a non-terminal, non-planned state — the work a
        restart has to reason about. Without ``direction`` this spans archives and
        restores, because either one half-done blocks the other."""
        with self._session_factory() as session:
            stmt = select(ArchiveOperation).where(
                ArchiveOperation.journal_state.not_in([*TERMINAL_STATES, ArchiveState.PLANNED])
            )
            if direction is not None:
                stmt = stmt.where(ArchiveOperation.direction == direction)
            rows = session.scalars(
                stmt.order_by(ArchiveOperation.plan_id, ArchiveOperation.sequence)
            )
            return [_operation_dict(row) for row in rows]

    # ── startup classification ────────────────────────────────────────────────

    def classify(self, operation_id: str) -> dict:
        """Classify one incomplete operation from the journal plus disk evidence.

        Hashes the archived copy when one exists: "a file is at the destination" is
        not evidence that the *right* bytes are, and only the right bytes justify
        removing an original. Never mutates anything.
        """
        row = self.get(operation_id)
        if row is None:
            raise JournalError(f"unknown archive operation {operation_id!r}")
        source = Path(row["source_path"])
        destination = Path(row["destination_path"])
        source_exists = source.exists()
        destination_exists = destination.exists()
        destination_matches = (
            destination_exists and sha256_file(destination) == row["expected_sha256"]
        )

        classification, reason = _classify(
            row["journal_state"], source_exists, destination_exists, destination_matches
        )
        return {
            "operation_id": operation_id,
            "plan_id": row["plan_id"],
            "direction": row["direction"],
            "album": row["album"],
            "asset_id": row["asset_id"],
            "source_path": row["source_path"],
            "destination_path": row["destination_path"],
            "journal_state": row["journal_state"],
            "classification": classification,
            "reason": reason,
            "source_exists": source_exists,
            "destination_exists": destination_exists,
            "destination_matches": destination_matches,
        }

    def classify_all(self, *, direction: str | None = None) -> list[dict]:
        return [self.classify(row["id"]) for row in self.incomplete(direction=direction)]

    def blocks_mutation(self) -> bool:
        """True when any item may have the library half-archived."""
        return any(row["journal_state"] in UNSAFE_STATES for row in self.incomplete())

    # ── plan-level ────────────────────────────────────────────────────────────

    def plan_state(self, plan_id: str) -> str:
        """Derive the plan's state from its items, so the summary can never disagree
        with the journal."""
        states = {row["journal_state"] for row in self.operations(plan_id)}
        if not states:
            return "planned"
        if states <= {ArchiveState.COMPLETE}:
            return "complete"
        if states & {ArchiveState.FAILED}:
            return "failed"
        if states & UNSAFE_STATES:
            return "applying"
        return "planned"

    def sync_plan_state(self, plan_id: str) -> str:
        state = self.plan_state(plan_id)
        with self._session_factory() as session:
            plan = session.get(ArchivePlan, plan_id)
            if plan is None:
                raise JournalError(f"unknown archive plan {plan_id!r}")
            if plan.state != state:
                plan.state = state
                plan.version += 1
                plan.updated_at = _now()
                if state == "complete" and plan.completed_at is None:
                    plan.completed_at = _now()
                session.commit()
        return state

    @staticmethod
    def _require(session, operation_id: str) -> ArchiveOperation:
        row = session.get(ArchiveOperation, operation_id)
        if row is None:
            raise JournalError(f"unknown archive operation {operation_id!r}")
        return row


def _classify(
    state: str, source_exists: bool, destination_exists: bool, destination_matches: bool
) -> tuple[str, str]:
    """The evidence table. Kept a pure function so every combination is testable."""
    if destination_exists and not destination_matches and state != ArchiveState.PLANNED:
        # Someone else's file, or a partial/edited copy: never overwrite it, and
        # never treat it as the durable archive that justifies a deletion.
        return MANUAL, "the archived path holds bytes that are not the recorded ones"

    if state in (ArchiveState.TRANSFERRING, ArchiveState.FAILED):
        if destination_matches:
            return FORWARD, "the archived copy is durable; finish the remaining steps"
        if source_exists:
            return RESUMABLE, "nothing was published; the source is intact"
        return MANUAL, "neither the source nor a verified archive copy is present"

    if state in (ArchiveState.VERIFIED, ArchiveState.REMOVING):
        if destination_matches:
            return FORWARD, "the archived copy is durable; finish the remaining steps"
        return MANUAL, f"journal says {state} but the archived copy is missing"

    return MANUAL, f"unhandled journal state {state}"


def _operation_dict(row: ArchiveOperation) -> dict:
    return {
        "id": row.id,
        "plan_id": row.plan_id,
        "direction": row.direction,
        "sequence": row.sequence,
        "album": row.album,
        "asset_id": row.asset_id,
        "source_path": row.source_path,
        "destination_path": row.destination_path,
        "archive_path": row.archive_path,
        "expected_sha256": row.expected_sha256,
        "byte_size": row.byte_size,
        "same_filesystem": row.same_filesystem,
        "journal_state": row.journal_state,
        "attempt_count": row.attempt_count,
        "fencing_token": row.fencing_token,
        "worker_id": row.worker_id,
        "verified_at": row.verified_at.isoformat() if row.verified_at else None,
        "removed_at": row.removed_at.isoformat() if row.removed_at else None,
        "error_code": row.error_code,
        "error_message": row.error_message,
    }