258 lines
10 KiB
Python
258 lines
10 KiB
Python
"""InventoryService — discovery and reconciliation against stable asset identity.
|
|
|
|
A scan discovers eligible files through the shared path policy, hashes each one,
|
|
and reconciles them against the ``assets`` table so identity survives moves and
|
|
renames. Every discovered or absent path is classified as one occurrence:
|
|
|
|
- ``new`` — a file with no matching identity;
|
|
- ``moved`` — bytes match an asset whose old path is gone (id retained);
|
|
- ``copied`` — bytes match an asset still present (distinct new id);
|
|
- ``replaced`` — same path, different pixels;
|
|
- ``metadata_changed`` — same path and pixels, different bytes (e.g. EXIF edit);
|
|
- ``unchanged`` — same path and bytes;
|
|
- ``missing`` — a known active asset whose file is gone (kept, flagged).
|
|
|
|
Missing files are never pruned (that would break identity); the asset is retained
|
|
with ``missing_at`` set and its availability becomes ``missing_unexpected`` —
|
|
nothing explains where the bytes went. Archived assets are left untouched: their
|
|
absence from the active roots is expected, and each scan re-derives whether their
|
|
medium is reachable (:mod:`photo_pipeline.services.availability`). Rescanning
|
|
unchanged input makes no durable change.
|
|
|
|
Extracted from photo_analyzer.discover_photos/reconcile_moved/prune_missing
|
|
(see donor_ledger.yaml: pa-discovery, pa-prune-missing).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import uuid
|
|
from collections import Counter
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from enum import Enum
|
|
from pathlib import Path
|
|
from typing import Iterable
|
|
|
|
from sqlalchemy import func, or_, select
|
|
from sqlalchemy.orm import Session, sessionmaker
|
|
|
|
from photo_pipeline import path_policy
|
|
from photo_pipeline.models import Asset, AssetPath
|
|
from photo_pipeline.services import availability, hashing
|
|
|
|
|
|
class Occurrence(str, Enum):
|
|
NEW = "new"
|
|
MOVED = "moved"
|
|
COPIED = "copied"
|
|
REPLACED = "replaced"
|
|
METADATA_CHANGED = "metadata_changed"
|
|
UNCHANGED = "unchanged"
|
|
MISSING = "missing"
|
|
|
|
|
|
@dataclass
|
|
class ScanResult:
|
|
counts: dict[str, int] = field(default_factory=dict)
|
|
occurrences: dict[str, str] = field(default_factory=dict) # path -> occurrence
|
|
asset_ids: dict[str, str] = field(default_factory=dict) # path -> asset id
|
|
|
|
|
|
def _asset_dict(asset: Asset) -> dict:
|
|
return {
|
|
"id": asset.id,
|
|
"current_path": asset.current_path,
|
|
"availability_state": asset.availability_state,
|
|
"archive_location_id": asset.archive_location_id,
|
|
"archive_path": asset.archive_path,
|
|
"byte_size": asset.byte_size,
|
|
"current_sha256": asset.current_sha256,
|
|
"pixel_sha256": asset.pixel_sha256,
|
|
"phash": asset.phash,
|
|
"canonical_asset_id": asset.canonical_asset_id,
|
|
"missing": asset.missing_at is not None,
|
|
"state_version": asset.state_version,
|
|
}
|
|
|
|
|
|
class InventoryService:
|
|
def __init__(self, session_factory: sessionmaker) -> None:
|
|
self._session_factory = session_factory
|
|
|
|
def scan(self, roots: Iterable[Path | str] | Path | str) -> ScanResult:
|
|
if isinstance(roots, (str, Path)):
|
|
roots = [roots]
|
|
normalized_roots = [path_policy.normalize_root(r) for r in roots]
|
|
discovered = path_policy.discover(normalized_roots)
|
|
discovered_paths = {str(p) for p in discovered}
|
|
now = datetime.now(timezone.utc)
|
|
|
|
result = ScanResult()
|
|
with self._session_factory() as session:
|
|
assets = list(session.execute(select(Asset)).scalars())
|
|
by_path = {a.current_path: a for a in assets if a.current_path}
|
|
by_sha: dict[str, list[Asset]] = {}
|
|
for asset in assets:
|
|
if asset.current_sha256:
|
|
by_sha.setdefault(asset.current_sha256, []).append(asset)
|
|
|
|
seen_ids: set[str] = set()
|
|
for path in discovered:
|
|
asset, occ = self._reconcile_one(
|
|
session, path, discovered_paths, by_path, by_sha, now
|
|
)
|
|
seen_ids.add(asset.id)
|
|
result.occurrences[str(path)] = occ.value
|
|
result.asset_ids[str(path)] = asset.id
|
|
|
|
for asset in assets:
|
|
# Archived assets are explained by their location, not by the active
|
|
# roots: a scan must never prune or flag them (concept §9).
|
|
if asset.availability_state in availability.ARCHIVED or asset.id in seen_ids:
|
|
continue
|
|
if asset.current_path and asset.current_path not in discovered_paths:
|
|
if not Path(asset.current_path).exists():
|
|
if asset.missing_at is None:
|
|
asset.missing_at = now
|
|
asset.state_version += 1
|
|
asset.updated_at = now
|
|
# Nothing explains this absence — it is not an offline medium.
|
|
if asset.availability_state != availability.MISSING_UNEXPECTED:
|
|
asset.availability_state = availability.MISSING_UNEXPECTED
|
|
asset.state_version += 1
|
|
asset.updated_at = now
|
|
result.occurrences[asset.current_path] = Occurrence.MISSING.value
|
|
|
|
session.commit()
|
|
|
|
# Media may have been mounted or removed since the last scan.
|
|
availability.refresh(self._session_factory)
|
|
|
|
result.counts = dict(Counter(result.occurrences.values()))
|
|
return result
|
|
|
|
def list_assets(
|
|
self,
|
|
*,
|
|
limit: int = 50,
|
|
offset: int = 0,
|
|
availability: str | None = None,
|
|
query: str | None = None,
|
|
) -> dict:
|
|
"""Paged, filterable asset listing for the inventory UI (read-only)."""
|
|
limit = max(1, min(limit, 200))
|
|
offset = max(0, offset)
|
|
with self._session_factory() as session:
|
|
stmt = select(Asset)
|
|
if availability:
|
|
stmt = stmt.where(Asset.availability_state == availability)
|
|
if query:
|
|
like = f"%{query}%"
|
|
# An archived asset has no active path; it is searched where it lives.
|
|
stmt = stmt.where(
|
|
or_(Asset.current_path.like(like), Asset.archive_path.like(like))
|
|
)
|
|
total = session.scalar(select(func.count()).select_from(stmt.subquery()))
|
|
rows = session.execute(
|
|
stmt.order_by(Asset.current_path).limit(limit).offset(offset)
|
|
).scalars()
|
|
items = [_asset_dict(a) for a in rows]
|
|
return {"items": items, "total": int(total or 0), "limit": limit, "offset": offset}
|
|
|
|
def _reconcile_one(
|
|
self,
|
|
session: Session,
|
|
path: Path,
|
|
discovered_paths: set[str],
|
|
by_path: dict[str, Asset],
|
|
by_sha: dict[str, list[Asset]],
|
|
now: datetime,
|
|
) -> tuple[Asset, Occurrence]:
|
|
path_str = str(path)
|
|
sha = hashing.sha256_file(path)
|
|
pixel = hashing.safe_pixel_sha256(path)
|
|
size = path.stat().st_size
|
|
|
|
existing = by_path.get(path_str)
|
|
if existing is not None:
|
|
if existing.current_sha256 == sha:
|
|
occ = Occurrence.UNCHANGED
|
|
else:
|
|
same_pixels = pixel is not None and existing.pixel_sha256 == pixel
|
|
occ = Occurrence.METADATA_CHANGED if same_pixels else Occurrence.REPLACED
|
|
existing.current_sha256 = sha
|
|
existing.pixel_sha256 = pixel
|
|
existing.byte_size = size
|
|
existing.state_version += 1
|
|
existing.updated_at = now
|
|
self._open_path(session, existing.id, path_str, now, occ.value)
|
|
if existing.missing_at is not None:
|
|
existing.missing_at = None
|
|
existing.availability_state = availability.ACTIVE
|
|
existing.state_version += 1
|
|
existing.updated_at = now
|
|
return existing, occ
|
|
|
|
candidates = by_sha.get(sha, [])
|
|
moved_from = next(
|
|
(
|
|
a
|
|
for a in candidates
|
|
if a.current_path
|
|
and a.current_path not in discovered_paths
|
|
and not Path(a.current_path).exists()
|
|
),
|
|
None,
|
|
)
|
|
if moved_from is not None:
|
|
self._close_path(session, moved_from.id, moved_from.current_path, now)
|
|
moved_from.current_path = path_str
|
|
moved_from.byte_size = size
|
|
moved_from.missing_at = None
|
|
moved_from.availability_state = availability.ACTIVE
|
|
moved_from.state_version += 1
|
|
moved_from.updated_at = now
|
|
self._open_path(session, moved_from.id, path_str, now, Occurrence.MOVED.value)
|
|
by_path[path_str] = moved_from
|
|
return moved_from, Occurrence.MOVED
|
|
|
|
occ = Occurrence.COPIED if candidates else Occurrence.NEW
|
|
asset = Asset(
|
|
id=str(uuid.uuid4()),
|
|
original_path=path_str,
|
|
current_path=path_str,
|
|
current_sha256=sha,
|
|
pixel_sha256=pixel,
|
|
hash_version=hashing.PIXEL_HASH_VERSION,
|
|
byte_size=size,
|
|
discovered_at=now,
|
|
availability_state="active",
|
|
state_version=1,
|
|
)
|
|
session.add(asset)
|
|
session.flush()
|
|
self._open_path(session, asset.id, path_str, now, occ.value)
|
|
by_sha.setdefault(sha, []).append(asset)
|
|
by_path[path_str] = asset
|
|
return asset, occ
|
|
|
|
@staticmethod
|
|
def _open_path(session: Session, asset_id: str, path: str, now: datetime, reason: str) -> None:
|
|
session.add(
|
|
AssetPath(asset_id=asset_id, path=path, valid_from=now, reason=reason)
|
|
)
|
|
|
|
@staticmethod
|
|
def _close_path(session: Session, asset_id: str, path: str | None, now: datetime) -> None:
|
|
if not path:
|
|
return
|
|
rows = session.execute(
|
|
select(AssetPath).where(
|
|
AssetPath.asset_id == asset_id,
|
|
AssetPath.path == path,
|
|
AssetPath.valid_until.is_(None),
|
|
)
|
|
).scalars()
|
|
for row in rows:
|
|
row.valid_until = now
|