US01-03: Discover and Reconcile Stable Assets #48

Merged
domverse merged 1 commits from us/US01-03-discover-and-reconcile-stable-assets into main 2026-07-15 16:47:09 +02:00
7 changed files with 666 additions and 0 deletions
Showing only changes of commit 09806badf9 - Show all commits

View File

@@ -0,0 +1,78 @@
"""Library boundary and exclusion policy — the single source used everywhere.
One function decides what is inside the library, what is a supported photo, and
what is excluded (`_IGNORE/` and generated thumbnail/cache directories). Symlinks
that resolve outside the library root are rejected rather than silently followed,
so discovery can never escape the configured boundary.
Extracted from photo_analyzer.discover_photos / SUPPORTED_EXTENSIONS
(see donor_ledger.yaml: pa-discovery).
"""
from __future__ import annotations
import os
from pathlib import Path
from typing import Iterable, Iterator
SUPPORTED_EXTENSIONS = {
".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif", ".tiff", ".tif"
}
EXCLUDED_DIR_NAMES = {"_IGNORE", ".@__thumb"}
class PathPolicyError(ValueError):
"""A path violates the library boundary (e.g. a symlink escaping the root)."""
def is_supported(path: os.PathLike | str) -> bool:
return Path(path).suffix.lower() in SUPPORTED_EXTENSIONS
def is_excluded(path: os.PathLike | str) -> bool:
return any(part in EXCLUDED_DIR_NAMES for part in Path(path).parts)
def normalize_root(root: os.PathLike | str) -> Path:
return Path(root).expanduser().resolve()
def resolve_within(root: Path, path: os.PathLike | str) -> Path:
"""Resolve ``path`` (following symlinks) and confirm it stays within ``root``.
Raises ``PathPolicyError`` if the resolved path escapes the root.
"""
root = Path(root).resolve()
resolved = Path(path).resolve()
if resolved != root and root not in resolved.parents:
raise PathPolicyError(f"path escapes library root {root}: {path} -> {resolved}")
return resolved
def iter_supported_files(root: os.PathLike | str) -> Iterator[Path]:
"""Yield supported, non-excluded files under ``root`` in deterministic order.
Excluded directories are never descended into. A symlinked file whose target
escapes the root raises ``PathPolicyError`` (unsafe paths fail rather than
leaking outside the library).
"""
root = normalize_root(root)
found: list[Path] = []
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
dirnames[:] = sorted(d for d in dirnames if d not in EXCLUDED_DIR_NAMES)
for name in filenames:
candidate = Path(dirpath) / name
if not is_supported(candidate) or is_excluded(candidate):
continue
if candidate.is_symlink():
resolve_within(root, candidate) # raises on escape
found.append(candidate)
return iter(sorted(found))
def discover(roots: Iterable[os.PathLike | str]) -> list[Path]:
"""Deterministic, de-duplicated discovery across one or more roots."""
seen: set[Path] = set()
for root in roots:
seen.update(iter_supported_files(root))
return sorted(seen)

View File

@@ -0,0 +1 @@
"""Application services — importable domain logic shared by API, worker, and CLI."""

View File

@@ -0,0 +1,47 @@
"""Content hashing for identity and reconciliation.
- ``sha256_file`` is exact byte identity: it drives move/copy/replace matching.
- ``pixel_sha256`` hashes the normalized decoded pixels (EXIF orientation applied,
converted to RGB), so an EXIF-only edit keeps the same value while a genuine
content change does not. That distinction separates "metadata changed" from
"replaced".
Refactored from photo_analyzer._sha1_file/_phash_image: SHA-256 replaces SHA-1 as
byte identity and a normalized pixel hash is added (see donor_ledger.yaml:
pa-hashing). Perceptual hashing stays with the duplicate engine (US01-04).
"""
from __future__ import annotations
import hashlib
from pathlib import Path
from PIL import Image, ImageOps
PIXEL_HASH_VERSION = 1
_CHUNK = 1 << 20
def sha256_file(path: Path | str) -> str:
digest = hashlib.sha256()
with open(path, "rb") as handle:
for chunk in iter(lambda: handle.read(_CHUNK), b""):
digest.update(chunk)
return digest.hexdigest()
def pixel_sha256(path: Path | str) -> str:
with Image.open(path) as image:
oriented = ImageOps.exif_transpose(image)
rgb = oriented.convert("RGB")
header = f"{PIXEL_HASH_VERSION}:{rgb.width}x{rgb.height}:".encode()
payload = header + rgb.tobytes()
return hashlib.sha256(payload).hexdigest()
def safe_pixel_sha256(path: Path | str) -> str | None:
"""``pixel_sha256`` but returns None for undecodable images instead of raising."""
try:
return pixel_sha256(path)
except Exception:
return None

View File

@@ -0,0 +1,197 @@
"""InventoryService — discovery and reconciliation against stable asset identity.
A scan discovers eligible files through the shared path policy, hashes each one,
and reconciles them against the ``assets`` table so identity survives moves and
renames. Every discovered or absent path is classified as one occurrence:
- ``new`` — a file with no matching identity;
- ``moved`` — bytes match an asset whose old path is gone (id retained);
- ``copied`` — bytes match an asset still present (distinct new id);
- ``replaced`` — same path, different pixels;
- ``metadata_changed`` — same path and pixels, different bytes (e.g. EXIF edit);
- ``unchanged`` — same path and bytes;
- ``missing`` — a known active asset whose file is gone (kept, flagged).
Missing files are never pruned (that would break identity); the asset is retained
with ``missing_at`` set. Archived assets are left untouched. Rescanning unchanged
input makes no durable change.
Extracted from photo_analyzer.discover_photos/reconcile_moved/prune_missing
(see donor_ledger.yaml: pa-discovery, pa-prune-missing).
"""
from __future__ import annotations
import uuid
from collections import Counter
from dataclasses import dataclass, field
from datetime import datetime, timezone
from enum import Enum
from pathlib import Path
from typing import Iterable
from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from photo_pipeline import path_policy
from photo_pipeline.models import Asset, AssetPath
from photo_pipeline.services import hashing
class Occurrence(str, Enum):
NEW = "new"
MOVED = "moved"
COPIED = "copied"
REPLACED = "replaced"
METADATA_CHANGED = "metadata_changed"
UNCHANGED = "unchanged"
MISSING = "missing"
@dataclass
class ScanResult:
counts: dict[str, int] = field(default_factory=dict)
occurrences: dict[str, str] = field(default_factory=dict) # path -> occurrence
asset_ids: dict[str, str] = field(default_factory=dict) # path -> asset id
class InventoryService:
def __init__(self, session_factory: sessionmaker) -> None:
self._session_factory = session_factory
def scan(self, roots: Iterable[Path | str] | Path | str) -> ScanResult:
if isinstance(roots, (str, Path)):
roots = [roots]
normalized_roots = [path_policy.normalize_root(r) for r in roots]
discovered = path_policy.discover(normalized_roots)
discovered_paths = {str(p) for p in discovered}
now = datetime.now(timezone.utc)
result = ScanResult()
with self._session_factory() as session:
assets = list(session.execute(select(Asset)).scalars())
by_path = {a.current_path: a for a in assets if a.current_path}
by_sha: dict[str, list[Asset]] = {}
for asset in assets:
if asset.current_sha256:
by_sha.setdefault(asset.current_sha256, []).append(asset)
seen_ids: set[str] = set()
for path in discovered:
asset, occ = self._reconcile_one(
session, path, discovered_paths, by_path, by_sha, now
)
seen_ids.add(asset.id)
result.occurrences[str(path)] = occ.value
result.asset_ids[str(path)] = asset.id
for asset in assets:
if asset.availability_state != "active" or asset.id in seen_ids:
continue
if asset.current_path and asset.current_path not in discovered_paths:
if not Path(asset.current_path).exists():
if asset.missing_at is None:
asset.missing_at = now
asset.state_version += 1
asset.updated_at = now
result.occurrences[asset.current_path] = Occurrence.MISSING.value
session.commit()
result.counts = dict(Counter(result.occurrences.values()))
return result
def _reconcile_one(
self,
session: Session,
path: Path,
discovered_paths: set[str],
by_path: dict[str, Asset],
by_sha: dict[str, list[Asset]],
now: datetime,
) -> tuple[Asset, Occurrence]:
path_str = str(path)
sha = hashing.sha256_file(path)
pixel = hashing.safe_pixel_sha256(path)
size = path.stat().st_size
existing = by_path.get(path_str)
if existing is not None:
if existing.current_sha256 == sha:
occ = Occurrence.UNCHANGED
else:
same_pixels = pixel is not None and existing.pixel_sha256 == pixel
occ = Occurrence.METADATA_CHANGED if same_pixels else Occurrence.REPLACED
existing.current_sha256 = sha
existing.pixel_sha256 = pixel
existing.byte_size = size
existing.state_version += 1
existing.updated_at = now
self._open_path(session, existing.id, path_str, now, occ.value)
if existing.missing_at is not None:
existing.missing_at = None
existing.state_version += 1
existing.updated_at = now
return existing, occ
candidates = by_sha.get(sha, [])
moved_from = next(
(
a
for a in candidates
if a.current_path
and a.current_path not in discovered_paths
and not Path(a.current_path).exists()
),
None,
)
if moved_from is not None:
self._close_path(session, moved_from.id, moved_from.current_path, now)
moved_from.current_path = path_str
moved_from.byte_size = size
moved_from.missing_at = None
moved_from.state_version += 1
moved_from.updated_at = now
self._open_path(session, moved_from.id, path_str, now, Occurrence.MOVED.value)
by_path[path_str] = moved_from
return moved_from, Occurrence.MOVED
occ = Occurrence.COPIED if candidates else Occurrence.NEW
asset = Asset(
id=str(uuid.uuid4()),
original_path=path_str,
current_path=path_str,
current_sha256=sha,
pixel_sha256=pixel,
hash_version=hashing.PIXEL_HASH_VERSION,
byte_size=size,
discovered_at=now,
availability_state="active",
state_version=1,
)
session.add(asset)
session.flush()
self._open_path(session, asset.id, path_str, now, occ.value)
by_sha.setdefault(sha, []).append(asset)
by_path[path_str] = asset
return asset, occ
@staticmethod
def _open_path(session: Session, asset_id: str, path: str, now: datetime, reason: str) -> None:
session.add(
AssetPath(asset_id=asset_id, path=path, valid_from=now, reason=reason)
)
@staticmethod
def _close_path(session: Session, asset_id: str, path: str | None, now: datetime) -> None:
if not path:
return
rows = session.execute(
select(AssetPath).where(
AssetPath.asset_id == asset_id,
AssetPath.path == path,
AssetPath.valid_until.is_(None),
)
).scalars()
for row in rows:
row.valid_until = now

View File

@@ -0,0 +1,212 @@
"""Reconciliation: stable identity across move/copy/replace/remove, with
idempotent rescans and durable state after a restart.
Fixtures live here rather than in a sibling conftest.py: the characterization
suite imports its own ``conftest`` by bare name, and a second sibling conftest
would shadow it under pytest's prepend import mode.
"""
import shutil
import numpy as np
import pytest
from PIL import Image
from sqlalchemy import select
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
from photo_pipeline.models import Asset, AssetPath
from photo_pipeline.services.inventory import InventoryService, Occurrence
@pytest.fixture
def db_url(tmp_path):
url = f"sqlite:///{tmp_path / 'inv.db'}"
run_migrations(url)
return url
@pytest.fixture
def make_factory():
engines = []
def _make(url):
engine = create_db_engine(url)
engines.append(engine)
return create_session_factory(engine)
yield _make
for engine in engines:
engine.dispose()
@pytest.fixture
def inventory(db_url, make_factory):
return InventoryService(make_factory(db_url))
@pytest.fixture
def make_image():
def _make(path, seed=1, size=(64, 48)):
path.parent.mkdir(parents=True, exist_ok=True)
rng = np.random.default_rng(seed)
arr = rng.integers(0, 256, (size[1], size[0], 3), dtype=np.uint8)
Image.fromarray(arr).save(path, quality=95)
return path
return _make
@pytest.fixture
def assets_by_id():
def _snapshot(factory):
with factory() as session:
return {a.id: a for a in session.execute(select(Asset)).scalars()}
return _snapshot
def test_initial_scan_registers_new_assets(tmp_path, inventory, make_image):
lib = tmp_path / "lib"
make_image(lib / "a.jpg", seed=1)
make_image(lib / "album" / "b.jpg", seed=2)
result = inventory.scan(lib)
assert result.counts.get(Occurrence.NEW.value) == 2
assert set(result.occurrences.values()) == {Occurrence.NEW.value}
assert len(result.asset_ids) == 2
def test_rescan_unchanged_is_idempotent(
tmp_path, inventory, db_url, make_factory, make_image, assets_by_id
):
lib = tmp_path / "lib"
make_image(lib / "a.jpg", seed=1)
make_image(lib / "b.jpg", seed=2)
first = inventory.scan(lib)
before = assets_by_id(make_factory(db_url))
second = inventory.scan(lib)
after = assets_by_id(make_factory(db_url))
assert set(second.occurrences.values()) == {Occurrence.UNCHANGED.value}
assert set(before) == set(after) # same asset ids, none created
assert first.asset_ids == second.asset_ids
def test_move_preserves_identity(tmp_path, inventory, db_url, make_factory, make_image):
lib = tmp_path / "lib"
src = make_image(lib / "a.jpg", seed=1)
original_id = next(iter(inventory.scan(lib).asset_ids.values()))
dst = lib / "album" / "moved.jpg"
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.move(str(src), str(dst))
result = inventory.scan(lib)
assert result.occurrences[str(dst)] == Occurrence.MOVED.value
assert result.asset_ids[str(dst)] == original_id # id retained across the move
# Path history: old occurrence closed, new occurrence open.
with make_factory(db_url)() as session:
paths = session.query(AssetPath).filter_by(asset_id=original_id).all()
by_path = {p.path: p for p in paths}
assert by_path[str(src)].valid_until is not None
assert by_path[str(dst)].valid_until is None
def test_copy_creates_distinct_identity(tmp_path, inventory, make_image):
lib = tmp_path / "lib"
src = make_image(lib / "a.jpg", seed=1)
first = inventory.scan(lib)
original_id = first.asset_ids[str(src)]
copy = lib / "a_copy.jpg"
shutil.copy2(str(src), str(copy))
result = inventory.scan(lib)
assert result.occurrences[str(copy)] == Occurrence.COPIED.value
assert result.asset_ids[str(copy)] != original_id
assert result.asset_ids[str(src)] == original_id
def test_replace_versus_metadata_change(tmp_path, inventory, make_image):
lib = tmp_path / "lib"
same = make_image(lib / "same.jpg", seed=1)
other = make_image(lib / "other.jpg", seed=2)
ids = inventory.scan(lib).asset_ids
# Metadata-style edit: append trailing bytes (pixels unchanged).
with open(same, "ab") as handle:
handle.write(b"\xff\xfe trailing")
# Content replacement: overwrite with different pixels.
make_image(other, seed=99)
result = inventory.scan(lib)
assert result.occurrences[str(same)] == Occurrence.METADATA_CHANGED.value
assert result.occurrences[str(other)] == Occurrence.REPLACED.value
# Identity survives both; hashes refreshed.
assert result.asset_ids[str(same)] == ids[str(same)]
assert result.asset_ids[str(other)] == ids[str(other)]
def test_missing_file_is_flagged_not_deleted(
tmp_path, inventory, db_url, make_factory, make_image, assets_by_id
):
lib = tmp_path / "lib"
a = make_image(lib / "a.jpg", seed=1)
make_image(lib / "b.jpg", seed=2)
ids = inventory.scan(lib).asset_ids
missing_id = ids[str(a)]
a.unlink()
result = inventory.scan(lib)
assert result.occurrences[str(a)] == Occurrence.MISSING.value
assets = assets_by_id(make_factory(db_url))
assert missing_id in assets # not pruned
assert assets[missing_id].missing_at is not None
assert assets[missing_id].availability_state == "active"
def test_reappearing_file_clears_missing(
tmp_path, inventory, db_url, make_factory, make_image, assets_by_id
):
lib = tmp_path / "lib"
a = make_image(lib / "a.jpg", seed=1)
ids = inventory.scan(lib).asset_ids
asset_id = ids[str(a)]
backup = tmp_path / "backup.jpg"
shutil.move(str(a), str(backup))
inventory.scan(lib) # marks missing
shutil.move(str(backup), str(a)) # reappears at the same path
inventory.scan(lib)
assets = assets_by_id(make_factory(db_url))
assert assets[asset_id].missing_at is None
def test_identity_and_state_durable_across_restart(
tmp_path, db_url, make_factory, make_image, assets_by_id
):
from photo_pipeline.services.inventory import InventoryService
lib = tmp_path / "lib"
src = make_image(lib / "a.jpg", seed=1)
service_a = InventoryService(make_factory(db_url))
original_id = service_a.scan(lib).asset_ids[str(src)]
# "Restart": a brand-new service/session factory against the same database.
dst = lib / "renamed.jpg"
shutil.move(str(src), str(dst))
service_b = InventoryService(make_factory(db_url))
result = service_b.scan(lib)
assert result.asset_ids[str(dst)] == original_id
assert result.occurrences[str(dst)] == Occurrence.MOVED.value
assets = assets_by_id(make_factory(db_url))
assert assets[original_id].current_path == str(dst)

View File

@@ -0,0 +1,48 @@
"""Hashing: byte identity vs normalized-pixel identity."""
import numpy as np
from PIL import Image
from photo_pipeline.services import hashing
def _write(path, seed, size=(48, 32)):
rng = np.random.default_rng(seed)
arr = rng.integers(0, 256, (size[1], size[0], 3), dtype=np.uint8)
Image.fromarray(arr).save(path, quality=95)
def test_sha256_is_deterministic_and_distinct(tmp_path):
a, b = tmp_path / "a.jpg", tmp_path / "b.jpg"
_write(a, 1)
_write(b, 2)
assert hashing.sha256_file(a) == hashing.sha256_file(a)
assert hashing.sha256_file(a) != hashing.sha256_file(b)
def test_trailing_bytes_change_sha_but_not_pixels(tmp_path):
"""Appending bytes after a JPEG (an EXIF-only-style edit) changes the file
hash while decoded pixels — and the pixel hash — stay the same."""
img = tmp_path / "a.jpg"
_write(img, 1)
sha_before = hashing.sha256_file(img)
pix_before = hashing.pixel_sha256(img)
with open(img, "ab") as handle:
handle.write(b"\xff\xfe trailing comment bytes")
assert hashing.sha256_file(img) != sha_before
assert hashing.pixel_sha256(img) == pix_before
def test_different_images_have_different_pixel_hashes(tmp_path):
a, b = tmp_path / "a.jpg", tmp_path / "b.jpg"
_write(a, 1)
_write(b, 2)
assert hashing.pixel_sha256(a) != hashing.pixel_sha256(b)
def test_safe_pixel_sha256_returns_none_for_garbage(tmp_path):
bad = tmp_path / "bad.jpg"
bad.write_bytes(b"not an image")
assert hashing.safe_pixel_sha256(bad) is None

View File

@@ -0,0 +1,83 @@
"""Path policy: supported extensions, exclusion, deterministic discovery, and
symlink-escape rejection."""
import os
import pytest
from photo_pipeline import path_policy
def test_is_supported_is_case_insensitive():
assert path_policy.is_supported("a.jpg")
assert path_policy.is_supported("a.JPG")
assert path_policy.is_supported("a.HEIC")
assert not path_policy.is_supported("a.txt")
assert not path_policy.is_supported("a.mp4")
def test_is_excluded_matches_ignore_and_thumb_anywhere():
assert path_policy.is_excluded("lib/_IGNORE/secret.jpg")
assert path_policy.is_excluded("lib/album/.@__thumb/t.jpg")
assert not path_policy.is_excluded("lib/album/photo.jpg")
def test_discovery_finds_supported_and_skips_excluded_and_unsupported(tmp_path):
(tmp_path / "album").mkdir()
(tmp_path / "_IGNORE").mkdir()
(tmp_path / "album" / ".@__thumb").mkdir()
keep = [tmp_path / "root.jpg", tmp_path / "album" / "a.PNG"]
for p in keep:
p.write_bytes(b"x")
(tmp_path / "notes.txt").write_bytes(b"x")
(tmp_path / "_IGNORE" / "secret.jpg").write_bytes(b"x")
(tmp_path / "album" / ".@__thumb" / "thumb.jpg").write_bytes(b"x")
found = path_policy.discover([tmp_path])
assert found == sorted(keep)
def test_discovery_is_deterministic(tmp_path):
for i in range(5):
(tmp_path / f"{i}.jpg").write_bytes(b"x")
assert path_policy.discover([tmp_path]) == path_policy.discover([tmp_path])
def test_resolve_within_accepts_inside_and_rejects_outside(tmp_path):
root = tmp_path / "lib"
root.mkdir()
inside = root / "a.jpg"
inside.write_bytes(b"x")
assert path_policy.resolve_within(root, inside) == inside.resolve()
with pytest.raises(path_policy.PathPolicyError):
path_policy.resolve_within(root, tmp_path / "outside.jpg")
@pytest.mark.skipif(not hasattr(os, "symlink"), reason="symlinks unsupported")
def test_symlink_escaping_root_fails_discovery(tmp_path):
root = tmp_path / "lib"
root.mkdir()
outside = tmp_path / "outside.jpg"
outside.write_bytes(b"x")
link = root / "link.jpg"
try:
os.symlink(outside, link)
except (OSError, NotImplementedError):
pytest.skip("cannot create symlink on this platform")
with pytest.raises(path_policy.PathPolicyError):
path_policy.discover([root])
@pytest.mark.skipif(not hasattr(os, "symlink"), reason="symlinks unsupported")
def test_symlink_within_root_is_allowed(tmp_path):
root = tmp_path / "lib"
root.mkdir()
target = root / "real.jpg"
target.write_bytes(b"x")
link = root / "alias.jpg"
try:
os.symlink(target, link)
except (OSError, NotImplementedError):
pytest.skip("cannot create symlink on this platform")
found = path_policy.discover([root])
assert target in found and link in found