606 lines
27 KiB
Python
606 lines
27 KiB
Python
"""Transfer an approved archive plan, verify it, and remove the active sources
|
|
(US06-02).
|
|
|
|
This is the only module that deletes originals from the photo library, so every
|
|
step exists to make one promise keepable: **a source is removed only after the
|
|
archived bytes are durable and proven identical.** The journal (US06-02,
|
|
:mod:`photo_pipeline.services.archive_journal`) records intent before each mutation;
|
|
this module performs the mutations and the recovery that reads that intent back.
|
|
|
|
Per file the sequence is:
|
|
|
|
```
|
|
journal.begin (transferring) ← intent persisted BEFORE any disk change
|
|
recheck preconditions ← source hash, free destination, no symlink
|
|
copy to a temporary file ← same directory, so the publish is atomic
|
|
fsync, close, hash it back ← read from disk; the write is not the evidence
|
|
atomically publish ← rename onto the final archive path
|
|
append the manifest entry ← durable on the medium itself, fsynced
|
|
journal → verified
|
|
journal → removing ← intent to delete, persisted first
|
|
re-verify the archive copy, remove the source, verify its absence
|
|
current_path=NULL, close the path occurrence, availability + location recorded
|
|
journal → complete
|
|
```
|
|
|
|
Same-filesystem albums may skip the copy and use an atomic ``rename`` instead
|
|
(concept §9), but only when the shared device is proven at run time — never from the
|
|
plan's stored guess — and the published file is hashed afterwards exactly as in the
|
|
copy path.
|
|
|
|
Rules that are never relaxed:
|
|
|
|
- An occupied destination is never overwritten; the item fails with the source
|
|
untouched.
|
|
- A source whose bytes no longer match the plan is never archived and never removed.
|
|
- A crash resolves from journal + disk evidence only: an archive copy that is
|
|
missing or hashes differently blocks the item for a human instead of being
|
|
retried or, worse, treated as a successful archive.
|
|
- Recovery is idempotent — repeated passes converge on the same state.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
from photo_pipeline.config import Config
|
|
from photo_pipeline.models import ArchiveLocation, ArchiveOperation, ArchivePlan, Asset, AssetPath
|
|
from photo_pipeline.services.archive_journal import (
|
|
MANUAL,
|
|
RESUMABLE,
|
|
ArchiveJournal,
|
|
ArchiveState,
|
|
)
|
|
from photo_pipeline.services.archives import MARKER_NAME, ArchiveError, ArchiveService
|
|
from photo_pipeline.services.hashing import sha256_file
|
|
from photo_pipeline.services.rename_apply import PreconditionFailed, maybe_fault
|
|
|
|
# The per-medium manifest: one JSON line per archived file, appended and fsynced
|
|
# before its source is removed. It lives with the bytes so the archive can still be
|
|
# read back if the database is lost.
|
|
MANIFEST_NAME = "archive-manifest.jsonl"
|
|
MANIFEST_VERSION = 1
|
|
TEMP_SUFFIX = ".part"
|
|
TEMP_PREFIX = ".archive-"
|
|
# Fault barrier between "the source is gone" and "the database knows it" — not a
|
|
# journal state, but the transition crash tests care about most.
|
|
SOURCE_REMOVED = "source_removed"
|
|
|
|
APPLYABLE_PLAN_STATES = frozenset({"planned", "applying", "failed", "complete"})
|
|
|
|
|
|
def _now() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
class ArchiveTransferService:
|
|
def __init__(self, session_factory: sessionmaker, *, config: Config) -> None:
|
|
self._session_factory = session_factory
|
|
self._config = config
|
|
self.journal = ArchiveJournal(session_factory)
|
|
|
|
# ── plans ─────────────────────────────────────────────────────────────────
|
|
|
|
def create(self, location_id: str, albums: list[str] | None = None, *, token: str) -> dict:
|
|
"""Turn an approved preflight into a durable plan.
|
|
|
|
The token is re-derived from a fresh preflight, so a plan can only be
|
|
created for the exact scope, bytes, and destination the user approved.
|
|
"""
|
|
preflight = ArchiveService(self._session_factory, config=self._config).preflight(
|
|
location_id, albums
|
|
)
|
|
if not token or token != preflight["token"]:
|
|
raise ArchiveError("stale_token", "the archive preflight changed since it was approved")
|
|
if preflight["state"] != "ready":
|
|
codes = ", ".join(sorted({issue["code"] for issue in preflight["blockers"]})) or "-"
|
|
raise ArchiveError("blocked", f"the archive scope is blocked: {codes}")
|
|
|
|
root = Path(preflight["location"]["root"])
|
|
plan_id = str(uuid.uuid4())
|
|
with self._session_factory() as session:
|
|
session.add(
|
|
ArchivePlan(
|
|
id=plan_id,
|
|
location_id=location_id,
|
|
token=token,
|
|
albums=json.dumps(albums) if albums is not None else None,
|
|
state="planned",
|
|
schema_version=MANIFEST_VERSION,
|
|
asset_count=preflight["totals"]["assets"],
|
|
byte_size=preflight["totals"]["bytes"],
|
|
)
|
|
)
|
|
session.flush() # the plan row must exist before its items reference it
|
|
sequence = 0
|
|
for album in preflight["albums"]:
|
|
destination_dir = Path(album["destination"])
|
|
for asset in album["assets"]:
|
|
source = Path(asset["current_path"])
|
|
destination = destination_dir / source.name
|
|
session.add(
|
|
ArchiveOperation(
|
|
id=str(uuid.uuid4()),
|
|
plan_id=plan_id,
|
|
sequence=sequence,
|
|
album=album["album"],
|
|
asset_id=asset["asset_id"],
|
|
source_path=str(source),
|
|
destination_path=str(destination),
|
|
archive_path=str(destination.relative_to(root)),
|
|
expected_sha256=asset["current_sha256"],
|
|
byte_size=asset["byte_size"],
|
|
# Recorded as a preview only; the real decision is made
|
|
# against the devices at apply time.
|
|
same_filesystem=album["transfer_method"] == "move",
|
|
journal_state=ArchiveState.PLANNED,
|
|
)
|
|
)
|
|
sequence += 1
|
|
session.commit()
|
|
return self.get(plan_id)
|
|
|
|
def get(self, plan_id: str) -> dict | None:
|
|
with self._session_factory() as session:
|
|
plan = session.get(ArchivePlan, plan_id)
|
|
if plan is None:
|
|
return None
|
|
report = _plan_dict(plan)
|
|
report["operations"] = self.journal.operations(plan_id)
|
|
return report
|
|
|
|
def list(self) -> list[dict]:
|
|
with self._session_factory() as session:
|
|
rows = session.scalars(select(ArchivePlan).order_by(ArchivePlan.created_at))
|
|
return [_plan_dict(row) for row in rows]
|
|
|
|
# ── apply ─────────────────────────────────────────────────────────────────
|
|
|
|
def apply(
|
|
self,
|
|
plan_id: str,
|
|
*,
|
|
expected_version: int | None = None,
|
|
worker_id: str = "archive",
|
|
) -> dict:
|
|
"""Archive every item of a plan, then report what happened.
|
|
|
|
Items are independent: one failure records its reason and leaves that
|
|
source in place; the rest of the album continues.
|
|
"""
|
|
plan = self._require_plan(plan_id)
|
|
if expected_version is not None and plan["version"] != expected_version:
|
|
raise ArchiveError(
|
|
"stale_plan",
|
|
f"plan {plan_id} is at version {plan['version']}, expected {expected_version}",
|
|
)
|
|
if plan["state"] not in APPLYABLE_PLAN_STATES:
|
|
raise ArchiveError("invalid_state", f"plan {plan_id} is {plan['state']}")
|
|
|
|
# One archiver lane: never start while another plan may be half-archived
|
|
# (concept §16 lock hierarchy).
|
|
blocking = [row for row in self.journal.incomplete() if row["plan_id"] != plan_id]
|
|
if blocking:
|
|
raise ArchiveError(
|
|
"archive_pending",
|
|
f"another archive operation is unresolved ({blocking[0]['id']}); recover it first",
|
|
)
|
|
|
|
token = self._claim_plan(plan_id)
|
|
location = self._location(plan["location_id"])
|
|
archived = failed = skipped = 0
|
|
for operation in self.journal.operations(plan_id):
|
|
if operation["journal_state"] == ArchiveState.COMPLETE:
|
|
skipped += 1 # repeated apply is a no-op for finished work
|
|
continue
|
|
try:
|
|
if operation["journal_state"] in (ArchiveState.VERIFIED, ArchiveState.REMOVING):
|
|
# The bytes are already archived; never transfer them twice.
|
|
self._finish(operation, location, token=token, worker_id=worker_id)
|
|
else:
|
|
self._archive_one(operation, location, token=token, worker_id=worker_id)
|
|
archived += 1
|
|
except PreconditionFailed as error:
|
|
self._fail(operation, token, error.code, str(error))
|
|
failed += 1
|
|
except Exception as error: # unexpected: record and stop touching disk
|
|
self._fail(operation, token, "archive_error", str(error))
|
|
failed += 1
|
|
self._prune_empty_sources(plan_id)
|
|
state = self.journal.sync_plan_state(plan_id)
|
|
return {
|
|
"plan_id": plan_id,
|
|
"archived": archived,
|
|
"failed": failed,
|
|
"skipped": skipped,
|
|
"state": state,
|
|
}
|
|
|
|
def _prune_empty_sources(self, plan_id: str) -> None:
|
|
"""Drop an album folder once every one of its files is archived.
|
|
|
|
``rmdir`` only: a folder that still holds anything at all — an unarchived
|
|
file, someone else's file, a subfolder — is left exactly as it is.
|
|
"""
|
|
folders: dict[Path, set[str]] = {}
|
|
for operation in self.journal.operations(plan_id):
|
|
folders.setdefault(Path(operation["source_path"]).parent, set()).add(
|
|
operation["journal_state"]
|
|
)
|
|
for folder, states in folders.items():
|
|
if states == {ArchiveState.COMPLETE}:
|
|
try:
|
|
folder.rmdir()
|
|
except OSError:
|
|
pass # not empty, or gone already; either way, leave it alone
|
|
|
|
def _archive_one(self, operation: dict, location: dict, *, token: int, worker_id: str) -> None:
|
|
source = Path(operation["source_path"])
|
|
destination = Path(operation["destination_path"])
|
|
|
|
# 1. Intent first — after this point a crash is recoverable from evidence.
|
|
self.journal.begin(operation["id"], worker_id=worker_id, fencing_token=token)
|
|
maybe_fault(ArchiveState.TRANSFERRING)
|
|
|
|
# 2. Recheck immediately before mutating; the plan's snapshot is not trusted.
|
|
self._recheck(operation, source, destination, location)
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
# 3. Transfer. Same-filesystem is an optimisation, so it has to be proven
|
|
# here rather than believed from the plan.
|
|
same_filesystem = _same_filesystem(source, destination.parent)
|
|
if same_filesystem:
|
|
os.rename(source, destination)
|
|
else:
|
|
self._copy_and_publish(operation, source, destination)
|
|
_fsync_dir(destination.parent)
|
|
|
|
# 4. The published file is the archive only once it hashes as recorded.
|
|
if sha256_file(destination) != operation["expected_sha256"]:
|
|
raise PreconditionFailed(
|
|
"archive_mismatch", f"{destination} does not hold the expected bytes"
|
|
)
|
|
_append_manifest(destination.parent, _manifest_entry(operation, location))
|
|
self.journal.transition(
|
|
operation["id"],
|
|
ArchiveState.VERIFIED,
|
|
fencing_token=token,
|
|
same_filesystem=same_filesystem,
|
|
)
|
|
maybe_fault(ArchiveState.VERIFIED)
|
|
|
|
# 5. Only now may the active source go.
|
|
self._finish(self.journal.get(operation["id"]), location, token=token, worker_id=worker_id)
|
|
|
|
def _copy_and_publish(self, operation: dict, source: Path, destination: Path) -> None:
|
|
"""Cross-filesystem: copy to a temporary file beside the destination, prove
|
|
its bytes, then publish it atomically. The source is still untouched."""
|
|
temp = destination.with_name(f"{TEMP_PREFIX}{uuid.uuid4().hex}{TEMP_SUFFIX}")
|
|
try:
|
|
with open(source, "rb") as src, open(temp, "wb") as out:
|
|
shutil.copyfileobj(src, out, 1024 * 1024)
|
|
out.flush()
|
|
os.fsync(out.fileno())
|
|
if sha256_file(temp) != operation["expected_sha256"]:
|
|
raise PreconditionFailed("copy_mismatch", f"{source} copied with wrong bytes")
|
|
if destination.exists():
|
|
raise PreconditionFailed(
|
|
"destination_exists", f"{destination} appeared during the transfer"
|
|
)
|
|
# ponytail: rename after an exists() check. The archiver lane is single
|
|
# and local; use O_EXCL/link-based publish if a second writer ever exists.
|
|
os.rename(temp, destination)
|
|
finally:
|
|
temp.unlink(missing_ok=True)
|
|
|
|
def _finish(self, operation: dict, location: dict, *, token: int, worker_id: str) -> None:
|
|
"""Drive an item whose archive copy is durable through removal and
|
|
bookkeeping. Every step is idempotent, so recovery may replay it."""
|
|
destination = Path(operation["destination_path"])
|
|
source = Path(operation["source_path"])
|
|
state = operation["journal_state"]
|
|
|
|
if state == ArchiveState.FAILED:
|
|
# The archive copy is durable even though the attempt ended badly:
|
|
# re-enter the transfer state so the remaining steps can run.
|
|
self.journal.transition(operation["id"], ArchiveState.TRANSFERRING, fencing_token=token)
|
|
state = ArchiveState.TRANSFERRING
|
|
|
|
if state == ArchiveState.TRANSFERRING:
|
|
if sha256_file(destination) != operation["expected_sha256"]:
|
|
raise PreconditionFailed(
|
|
"archive_mismatch", f"{destination} does not hold the expected bytes"
|
|
)
|
|
_append_manifest(destination.parent, _manifest_entry(operation, location))
|
|
self.journal.transition(operation["id"], ArchiveState.VERIFIED, fencing_token=token)
|
|
state = ArchiveState.VERIFIED
|
|
|
|
if state == ArchiveState.VERIFIED:
|
|
self.journal.transition(operation["id"], ArchiveState.REMOVING, fencing_token=token)
|
|
maybe_fault(ArchiveState.REMOVING)
|
|
state = ArchiveState.REMOVING
|
|
|
|
if state == ArchiveState.REMOVING:
|
|
# Re-verify the archived bytes immediately before deleting the original:
|
|
# this check is the entire justification for the removal.
|
|
if not destination.exists() or sha256_file(destination) != operation["expected_sha256"]:
|
|
raise PreconditionFailed(
|
|
"archive_unverified", f"{destination} is not a verified archive copy"
|
|
)
|
|
if source.exists():
|
|
if source.is_symlink():
|
|
raise PreconditionFailed("symlink", f"{source} became a symlink")
|
|
if sha256_file(source) != operation["expected_sha256"]:
|
|
raise PreconditionFailed(
|
|
"source_changed", f"{source} changed; it is not ours to remove"
|
|
)
|
|
source.unlink()
|
|
if source.exists():
|
|
raise PreconditionFailed("removal_failed", f"{source} is still present")
|
|
# The dangerous window: active storage no longer holds the file while the
|
|
# database still points at it.
|
|
maybe_fault(SOURCE_REMOVED)
|
|
self._record_archived(operation, location, destination)
|
|
self.journal.transition(operation["id"], ArchiveState.COMPLETE, fencing_token=token)
|
|
maybe_fault(ArchiveState.COMPLETE)
|
|
|
|
def _recheck(self, operation: dict, source: Path, destination: Path, location: dict) -> None:
|
|
if not source.exists():
|
|
raise PreconditionFailed("source_missing", f"source {source} disappeared")
|
|
if source.is_symlink() or destination.is_symlink():
|
|
raise PreconditionFailed("symlink", "refusing to archive through a symlink")
|
|
if destination.exists():
|
|
raise PreconditionFailed("destination_exists", f"destination {destination} is occupied")
|
|
root = Path(location["root"])
|
|
if root not in destination.parents:
|
|
raise PreconditionFailed(
|
|
"destination_escape", f"{destination} is outside the archive location {root}"
|
|
)
|
|
if not root.is_dir() or not (root / MARKER_NAME).exists():
|
|
raise PreconditionFailed("location_offline", f"{root} is not the archive medium")
|
|
if sha256_file(source) != operation["expected_sha256"]:
|
|
raise PreconditionFailed(
|
|
"source_changed", f"{source} changed since the plan was approved"
|
|
)
|
|
with self._session_factory() as session:
|
|
asset = session.get(Asset, operation["asset_id"])
|
|
if asset is None or asset.current_path != str(source):
|
|
raise PreconditionFailed(
|
|
"asset_moved", f"asset {operation['asset_id']} is no longer at {source}"
|
|
)
|
|
|
|
# ── database ──────────────────────────────────────────────────────────────
|
|
|
|
def _record_archived(self, operation: dict, location: dict, destination: Path) -> None:
|
|
"""The original is gone from active storage: drop ``current_path``, close its
|
|
occurrence, record where the bytes now live, and set availability."""
|
|
now = _now()
|
|
with self._session_factory() as session:
|
|
asset = session.get(Asset, operation["asset_id"])
|
|
if asset is None:
|
|
raise PreconditionFailed(
|
|
"asset_missing", f"asset {operation['asset_id']} no longer exists"
|
|
)
|
|
if asset.current_path:
|
|
for row in session.scalars(
|
|
select(AssetPath).where(
|
|
AssetPath.asset_id == asset.id,
|
|
AssetPath.path == asset.current_path,
|
|
AssetPath.valid_until.is_(None),
|
|
)
|
|
):
|
|
row.valid_until = now
|
|
recorded = session.scalar(
|
|
select(AssetPath).where(
|
|
AssetPath.asset_id == asset.id, AssetPath.path == str(destination)
|
|
)
|
|
)
|
|
if recorded is None: # idempotent: recovery may replay this
|
|
session.add(
|
|
AssetPath(
|
|
asset_id=asset.id,
|
|
path=str(destination),
|
|
valid_from=now,
|
|
reason="archive",
|
|
)
|
|
)
|
|
asset.current_path = None
|
|
asset.availability_state = (
|
|
"archived_online" if destination.exists() else "archived_offline"
|
|
)
|
|
asset.archive_location_id = location["id"]
|
|
asset.archive_path = operation["archive_path"]
|
|
asset.state_version += 1
|
|
asset.updated_at = now
|
|
session.commit()
|
|
|
|
# ── recovery ──────────────────────────────────────────────────────────────
|
|
|
|
def recover(self, *, worker_id: str = "archive-recovery") -> dict:
|
|
"""Resolve every incomplete item from journal + disk evidence.
|
|
|
|
Idempotent: running it repeatedly converges. Ambiguous (``manual``) work is
|
|
left exactly as found and keeps blocking unrelated mutations.
|
|
"""
|
|
results = {"resumed": 0, "completed": 0, "manual": 0}
|
|
touched: set[str] = set()
|
|
for verdict in self.journal.classify_all():
|
|
operation = self.journal.get(verdict["operation_id"])
|
|
touched.add(operation["plan_id"])
|
|
token = (operation["fencing_token"] or 0) + 1
|
|
if verdict["classification"] == MANUAL:
|
|
results["manual"] += 1
|
|
continue
|
|
if verdict["classification"] == RESUMABLE:
|
|
# Nothing was published: discard the debris and let a later apply
|
|
# retry the item cleanly.
|
|
_clean_temp_files(Path(operation["destination_path"]).parent)
|
|
self.journal.transition(operation["id"], ArchiveState.PLANNED, fencing_token=token)
|
|
results["resumed"] += 1
|
|
continue
|
|
location = self._location(self._require_plan(operation["plan_id"])["location_id"])
|
|
try:
|
|
self._finish(operation, location, token=token, worker_id=worker_id)
|
|
results["completed"] += 1
|
|
except PreconditionFailed as error:
|
|
self._fail(operation, token, error.code, str(error))
|
|
results["manual"] += 1
|
|
for plan_id in touched:
|
|
self._prune_empty_sources(plan_id)
|
|
self.journal.sync_plan_state(plan_id)
|
|
return results
|
|
|
|
def recovery_status(self) -> dict:
|
|
verdicts = self.journal.classify_all()
|
|
return {
|
|
"operations": verdicts,
|
|
"manual": [v for v in verdicts if v["classification"] == MANUAL],
|
|
"blocks_mutation": self.journal.blocks_mutation(),
|
|
}
|
|
|
|
# ── helpers ───────────────────────────────────────────────────────────────
|
|
|
|
def _fail(self, operation: dict, token: int, code: str, message: str) -> None:
|
|
self.journal.transition(
|
|
operation["id"], ArchiveState.FAILED, fencing_token=token, error=(code, message)
|
|
)
|
|
|
|
def _require_plan(self, plan_id: str) -> dict:
|
|
with self._session_factory() as session:
|
|
plan = session.get(ArchivePlan, plan_id)
|
|
if plan is None:
|
|
raise ArchiveError("unknown_plan", f"unknown archive plan {plan_id!r}")
|
|
return _plan_dict(plan)
|
|
|
|
def _location(self, location_id: str) -> dict:
|
|
with self._session_factory() as session:
|
|
location = session.get(ArchiveLocation, location_id)
|
|
if location is None:
|
|
raise ArchiveError("unknown_location", f"unknown archive location {location_id!r}")
|
|
return {"id": location.id, "root": location.root, "media_id": location.media_id}
|
|
|
|
def _claim_plan(self, plan_id: str) -> int:
|
|
"""Bump the plan version and use it as this attempt's fencing token, so a
|
|
worker from a superseded attempt cannot commit."""
|
|
with self._session_factory() as session:
|
|
plan = session.get(ArchivePlan, plan_id)
|
|
plan.version += 1
|
|
plan.state = "applying"
|
|
plan.updated_at = _now()
|
|
token = plan.version
|
|
session.commit()
|
|
return token
|
|
|
|
|
|
# ── module helpers ───────────────────────────────────────────────────────────
|
|
|
|
|
|
def _same_filesystem(source: Path, destination_dir: Path) -> bool:
|
|
"""Proven at run time from the actual devices, never from the plan's preview."""
|
|
try:
|
|
return source.stat().st_dev == destination_dir.stat().st_dev
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def _fsync_dir(path: Path) -> None:
|
|
"""Make the directory entry itself durable, so the published name survives a
|
|
power loss and not just the file's data."""
|
|
fd = os.open(path, os.O_RDONLY)
|
|
try:
|
|
os.fsync(fd)
|
|
except OSError:
|
|
pass # some filesystems refuse directory fsync; the data is already synced
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def _manifest_entry(operation: dict, location: dict) -> dict:
|
|
return {
|
|
"schema_version": MANIFEST_VERSION,
|
|
"plan_id": operation["plan_id"],
|
|
"asset_id": operation["asset_id"],
|
|
"album": operation["album"],
|
|
"archive_path": operation["archive_path"],
|
|
"source_path": operation["source_path"],
|
|
"sha256": operation["expected_sha256"],
|
|
"byte_size": operation["byte_size"],
|
|
"media_id": location["media_id"],
|
|
"archived_at": _now().isoformat(),
|
|
}
|
|
|
|
|
|
def _append_manifest(directory: Path, entry: dict) -> None:
|
|
"""Append one durable manifest line, skipping an entry that is already there.
|
|
|
|
The manifest is written before the source is removed, so it is the medium's own
|
|
record of what it holds even if the database is lost.
|
|
"""
|
|
path = directory / MANIFEST_NAME
|
|
# ponytail: rereads the album manifest per file (O(n²) lines for one album).
|
|
# Keep an in-memory index per plan if an album ever holds enough files to matter.
|
|
if path.exists():
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
try:
|
|
existing = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
if (existing.get("asset_id"), existing.get("sha256")) == (
|
|
entry["asset_id"],
|
|
entry["sha256"],
|
|
):
|
|
return
|
|
with open(path, "a", encoding="utf-8") as handle:
|
|
handle.write(json.dumps(entry, sort_keys=True) + "\n")
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
_fsync_dir(directory)
|
|
|
|
|
|
def read_manifest(directory: Path) -> list[dict]:
|
|
"""Every manifest entry an archived album directory holds."""
|
|
path = directory / MANIFEST_NAME
|
|
if not path.exists():
|
|
return []
|
|
entries = []
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
try:
|
|
entries.append(json.loads(line))
|
|
except ValueError:
|
|
continue
|
|
return entries
|
|
|
|
|
|
def _clean_temp_files(directory: Path) -> None:
|
|
"""Remove this application's own abandoned transfer temporaries — never any
|
|
other file (concept §17: startup cleans only recognised stale temporaries)."""
|
|
if not directory.is_dir():
|
|
return
|
|
for temp in directory.glob(f"{TEMP_PREFIX}*{TEMP_SUFFIX}"):
|
|
temp.unlink(missing_ok=True)
|
|
|
|
|
|
def _plan_dict(plan: ArchivePlan) -> dict:
|
|
return {
|
|
"id": plan.id,
|
|
"location_id": plan.location_id,
|
|
"token": plan.token,
|
|
"albums": json.loads(plan.albums) if plan.albums else None,
|
|
"state": plan.state,
|
|
"schema_version": plan.schema_version,
|
|
"asset_count": plan.asset_count,
|
|
"byte_size": plan.byte_size,
|
|
"version": plan.version,
|
|
"worker_id": plan.worker_id,
|
|
"completed_at": plan.completed_at.isoformat() if plan.completed_at else None,
|
|
"created_at": plan.created_at.isoformat() if plan.created_at else None,
|
|
}
|