US01-03: Discover and Reconcile Stable Assets #48
78
photo_pipeline/path_policy.py
Normal file
78
photo_pipeline/path_policy.py
Normal file
@@ -0,0 +1,78 @@
|
||||
"""Library boundary and exclusion policy — the single source used everywhere.
|
||||
|
||||
One function decides what is inside the library, what is a supported photo, and
|
||||
what is excluded (`_IGNORE/` and generated thumbnail/cache directories). Symlinks
|
||||
that resolve outside the library root are rejected rather than silently followed,
|
||||
so discovery can never escape the configured boundary.
|
||||
|
||||
Extracted from photo_analyzer.discover_photos / SUPPORTED_EXTENSIONS
|
||||
(see donor_ledger.yaml: pa-discovery).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Iterator
|
||||
|
||||
SUPPORTED_EXTENSIONS = {
|
||||
".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif", ".tiff", ".tif"
|
||||
}
|
||||
EXCLUDED_DIR_NAMES = {"_IGNORE", ".@__thumb"}
|
||||
|
||||
|
||||
class PathPolicyError(ValueError):
|
||||
"""A path violates the library boundary (e.g. a symlink escaping the root)."""
|
||||
|
||||
|
||||
def is_supported(path: os.PathLike | str) -> bool:
|
||||
return Path(path).suffix.lower() in SUPPORTED_EXTENSIONS
|
||||
|
||||
|
||||
def is_excluded(path: os.PathLike | str) -> bool:
|
||||
return any(part in EXCLUDED_DIR_NAMES for part in Path(path).parts)
|
||||
|
||||
|
||||
def normalize_root(root: os.PathLike | str) -> Path:
|
||||
return Path(root).expanduser().resolve()
|
||||
|
||||
|
||||
def resolve_within(root: Path, path: os.PathLike | str) -> Path:
|
||||
"""Resolve ``path`` (following symlinks) and confirm it stays within ``root``.
|
||||
|
||||
Raises ``PathPolicyError`` if the resolved path escapes the root.
|
||||
"""
|
||||
root = Path(root).resolve()
|
||||
resolved = Path(path).resolve()
|
||||
if resolved != root and root not in resolved.parents:
|
||||
raise PathPolicyError(f"path escapes library root {root}: {path} -> {resolved}")
|
||||
return resolved
|
||||
|
||||
|
||||
def iter_supported_files(root: os.PathLike | str) -> Iterator[Path]:
|
||||
"""Yield supported, non-excluded files under ``root`` in deterministic order.
|
||||
|
||||
Excluded directories are never descended into. A symlinked file whose target
|
||||
escapes the root raises ``PathPolicyError`` (unsafe paths fail rather than
|
||||
leaking outside the library).
|
||||
"""
|
||||
root = normalize_root(root)
|
||||
found: list[Path] = []
|
||||
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
|
||||
dirnames[:] = sorted(d for d in dirnames if d not in EXCLUDED_DIR_NAMES)
|
||||
for name in filenames:
|
||||
candidate = Path(dirpath) / name
|
||||
if not is_supported(candidate) or is_excluded(candidate):
|
||||
continue
|
||||
if candidate.is_symlink():
|
||||
resolve_within(root, candidate) # raises on escape
|
||||
found.append(candidate)
|
||||
return iter(sorted(found))
|
||||
|
||||
|
||||
def discover(roots: Iterable[os.PathLike | str]) -> list[Path]:
|
||||
"""Deterministic, de-duplicated discovery across one or more roots."""
|
||||
seen: set[Path] = set()
|
||||
for root in roots:
|
||||
seen.update(iter_supported_files(root))
|
||||
return sorted(seen)
|
||||
1
photo_pipeline/services/__init__.py
Normal file
1
photo_pipeline/services/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Application services — importable domain logic shared by API, worker, and CLI."""
|
||||
47
photo_pipeline/services/hashing.py
Normal file
47
photo_pipeline/services/hashing.py
Normal file
@@ -0,0 +1,47 @@
|
||||
"""Content hashing for identity and reconciliation.
|
||||
|
||||
- ``sha256_file`` is exact byte identity: it drives move/copy/replace matching.
|
||||
- ``pixel_sha256`` hashes the normalized decoded pixels (EXIF orientation applied,
|
||||
converted to RGB), so an EXIF-only edit keeps the same value while a genuine
|
||||
content change does not. That distinction separates "metadata changed" from
|
||||
"replaced".
|
||||
|
||||
Refactored from photo_analyzer._sha1_file/_phash_image: SHA-256 replaces SHA-1 as
|
||||
byte identity and a normalized pixel hash is added (see donor_ledger.yaml:
|
||||
pa-hashing). Perceptual hashing stays with the duplicate engine (US01-04).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
PIXEL_HASH_VERSION = 1
|
||||
_CHUNK = 1 << 20
|
||||
|
||||
|
||||
def sha256_file(path: Path | str) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(_CHUNK), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def pixel_sha256(path: Path | str) -> str:
|
||||
with Image.open(path) as image:
|
||||
oriented = ImageOps.exif_transpose(image)
|
||||
rgb = oriented.convert("RGB")
|
||||
header = f"{PIXEL_HASH_VERSION}:{rgb.width}x{rgb.height}:".encode()
|
||||
payload = header + rgb.tobytes()
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def safe_pixel_sha256(path: Path | str) -> str | None:
|
||||
"""``pixel_sha256`` but returns None for undecodable images instead of raising."""
|
||||
try:
|
||||
return pixel_sha256(path)
|
||||
except Exception:
|
||||
return None
|
||||
197
photo_pipeline/services/inventory.py
Normal file
197
photo_pipeline/services/inventory.py
Normal file
@@ -0,0 +1,197 @@
|
||||
"""InventoryService — discovery and reconciliation against stable asset identity.
|
||||
|
||||
A scan discovers eligible files through the shared path policy, hashes each one,
|
||||
and reconciles them against the ``assets`` table so identity survives moves and
|
||||
renames. Every discovered or absent path is classified as one occurrence:
|
||||
|
||||
- ``new`` — a file with no matching identity;
|
||||
- ``moved`` — bytes match an asset whose old path is gone (id retained);
|
||||
- ``copied`` — bytes match an asset still present (distinct new id);
|
||||
- ``replaced`` — same path, different pixels;
|
||||
- ``metadata_changed`` — same path and pixels, different bytes (e.g. EXIF edit);
|
||||
- ``unchanged`` — same path and bytes;
|
||||
- ``missing`` — a known active asset whose file is gone (kept, flagged).
|
||||
|
||||
Missing files are never pruned (that would break identity); the asset is retained
|
||||
with ``missing_at`` set. Archived assets are left untouched. Rescanning unchanged
|
||||
input makes no durable change.
|
||||
|
||||
Extracted from photo_analyzer.discover_photos/reconcile_moved/prune_missing
|
||||
(see donor_ledger.yaml: pa-discovery, pa-prune-missing).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from photo_pipeline import path_policy
|
||||
from photo_pipeline.models import Asset, AssetPath
|
||||
from photo_pipeline.services import hashing
|
||||
|
||||
|
||||
class Occurrence(str, Enum):
|
||||
NEW = "new"
|
||||
MOVED = "moved"
|
||||
COPIED = "copied"
|
||||
REPLACED = "replaced"
|
||||
METADATA_CHANGED = "metadata_changed"
|
||||
UNCHANGED = "unchanged"
|
||||
MISSING = "missing"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanResult:
|
||||
counts: dict[str, int] = field(default_factory=dict)
|
||||
occurrences: dict[str, str] = field(default_factory=dict) # path -> occurrence
|
||||
asset_ids: dict[str, str] = field(default_factory=dict) # path -> asset id
|
||||
|
||||
|
||||
class InventoryService:
|
||||
def __init__(self, session_factory: sessionmaker) -> None:
|
||||
self._session_factory = session_factory
|
||||
|
||||
def scan(self, roots: Iterable[Path | str] | Path | str) -> ScanResult:
|
||||
if isinstance(roots, (str, Path)):
|
||||
roots = [roots]
|
||||
normalized_roots = [path_policy.normalize_root(r) for r in roots]
|
||||
discovered = path_policy.discover(normalized_roots)
|
||||
discovered_paths = {str(p) for p in discovered}
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
result = ScanResult()
|
||||
with self._session_factory() as session:
|
||||
assets = list(session.execute(select(Asset)).scalars())
|
||||
by_path = {a.current_path: a for a in assets if a.current_path}
|
||||
by_sha: dict[str, list[Asset]] = {}
|
||||
for asset in assets:
|
||||
if asset.current_sha256:
|
||||
by_sha.setdefault(asset.current_sha256, []).append(asset)
|
||||
|
||||
seen_ids: set[str] = set()
|
||||
for path in discovered:
|
||||
asset, occ = self._reconcile_one(
|
||||
session, path, discovered_paths, by_path, by_sha, now
|
||||
)
|
||||
seen_ids.add(asset.id)
|
||||
result.occurrences[str(path)] = occ.value
|
||||
result.asset_ids[str(path)] = asset.id
|
||||
|
||||
for asset in assets:
|
||||
if asset.availability_state != "active" or asset.id in seen_ids:
|
||||
continue
|
||||
if asset.current_path and asset.current_path not in discovered_paths:
|
||||
if not Path(asset.current_path).exists():
|
||||
if asset.missing_at is None:
|
||||
asset.missing_at = now
|
||||
asset.state_version += 1
|
||||
asset.updated_at = now
|
||||
result.occurrences[asset.current_path] = Occurrence.MISSING.value
|
||||
|
||||
session.commit()
|
||||
|
||||
result.counts = dict(Counter(result.occurrences.values()))
|
||||
return result
|
||||
|
||||
def _reconcile_one(
|
||||
self,
|
||||
session: Session,
|
||||
path: Path,
|
||||
discovered_paths: set[str],
|
||||
by_path: dict[str, Asset],
|
||||
by_sha: dict[str, list[Asset]],
|
||||
now: datetime,
|
||||
) -> tuple[Asset, Occurrence]:
|
||||
path_str = str(path)
|
||||
sha = hashing.sha256_file(path)
|
||||
pixel = hashing.safe_pixel_sha256(path)
|
||||
size = path.stat().st_size
|
||||
|
||||
existing = by_path.get(path_str)
|
||||
if existing is not None:
|
||||
if existing.current_sha256 == sha:
|
||||
occ = Occurrence.UNCHANGED
|
||||
else:
|
||||
same_pixels = pixel is not None and existing.pixel_sha256 == pixel
|
||||
occ = Occurrence.METADATA_CHANGED if same_pixels else Occurrence.REPLACED
|
||||
existing.current_sha256 = sha
|
||||
existing.pixel_sha256 = pixel
|
||||
existing.byte_size = size
|
||||
existing.state_version += 1
|
||||
existing.updated_at = now
|
||||
self._open_path(session, existing.id, path_str, now, occ.value)
|
||||
if existing.missing_at is not None:
|
||||
existing.missing_at = None
|
||||
existing.state_version += 1
|
||||
existing.updated_at = now
|
||||
return existing, occ
|
||||
|
||||
candidates = by_sha.get(sha, [])
|
||||
moved_from = next(
|
||||
(
|
||||
a
|
||||
for a in candidates
|
||||
if a.current_path
|
||||
and a.current_path not in discovered_paths
|
||||
and not Path(a.current_path).exists()
|
||||
),
|
||||
None,
|
||||
)
|
||||
if moved_from is not None:
|
||||
self._close_path(session, moved_from.id, moved_from.current_path, now)
|
||||
moved_from.current_path = path_str
|
||||
moved_from.byte_size = size
|
||||
moved_from.missing_at = None
|
||||
moved_from.state_version += 1
|
||||
moved_from.updated_at = now
|
||||
self._open_path(session, moved_from.id, path_str, now, Occurrence.MOVED.value)
|
||||
by_path[path_str] = moved_from
|
||||
return moved_from, Occurrence.MOVED
|
||||
|
||||
occ = Occurrence.COPIED if candidates else Occurrence.NEW
|
||||
asset = Asset(
|
||||
id=str(uuid.uuid4()),
|
||||
original_path=path_str,
|
||||
current_path=path_str,
|
||||
current_sha256=sha,
|
||||
pixel_sha256=pixel,
|
||||
hash_version=hashing.PIXEL_HASH_VERSION,
|
||||
byte_size=size,
|
||||
discovered_at=now,
|
||||
availability_state="active",
|
||||
state_version=1,
|
||||
)
|
||||
session.add(asset)
|
||||
session.flush()
|
||||
self._open_path(session, asset.id, path_str, now, occ.value)
|
||||
by_sha.setdefault(sha, []).append(asset)
|
||||
by_path[path_str] = asset
|
||||
return asset, occ
|
||||
|
||||
@staticmethod
|
||||
def _open_path(session: Session, asset_id: str, path: str, now: datetime, reason: str) -> None:
|
||||
session.add(
|
||||
AssetPath(asset_id=asset_id, path=path, valid_from=now, reason=reason)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _close_path(session: Session, asset_id: str, path: str | None, now: datetime) -> None:
|
||||
if not path:
|
||||
return
|
||||
rows = session.execute(
|
||||
select(AssetPath).where(
|
||||
AssetPath.asset_id == asset_id,
|
||||
AssetPath.path == path,
|
||||
AssetPath.valid_until.is_(None),
|
||||
)
|
||||
).scalars()
|
||||
for row in rows:
|
||||
row.valid_until = now
|
||||
212
tests/integration/test_inventory_reconcile.py
Normal file
212
tests/integration/test_inventory_reconcile.py
Normal file
@@ -0,0 +1,212 @@
|
||||
"""Reconciliation: stable identity across move/copy/replace/remove, with
|
||||
idempotent rescans and durable state after a restart.
|
||||
|
||||
Fixtures live here rather than in a sibling conftest.py: the characterization
|
||||
suite imports its own ``conftest`` by bare name, and a second sibling conftest
|
||||
would shadow it under pytest's prepend import mode.
|
||||
"""
|
||||
|
||||
import shutil
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from sqlalchemy import select
|
||||
|
||||
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
|
||||
from photo_pipeline.models import Asset, AssetPath
|
||||
from photo_pipeline.services.inventory import InventoryService, Occurrence
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db_url(tmp_path):
|
||||
url = f"sqlite:///{tmp_path / 'inv.db'}"
|
||||
run_migrations(url)
|
||||
return url
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def make_factory():
|
||||
engines = []
|
||||
|
||||
def _make(url):
|
||||
engine = create_db_engine(url)
|
||||
engines.append(engine)
|
||||
return create_session_factory(engine)
|
||||
|
||||
yield _make
|
||||
for engine in engines:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def inventory(db_url, make_factory):
|
||||
return InventoryService(make_factory(db_url))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def make_image():
|
||||
def _make(path, seed=1, size=(64, 48)):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
rng = np.random.default_rng(seed)
|
||||
arr = rng.integers(0, 256, (size[1], size[0], 3), dtype=np.uint8)
|
||||
Image.fromarray(arr).save(path, quality=95)
|
||||
return path
|
||||
|
||||
return _make
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def assets_by_id():
|
||||
def _snapshot(factory):
|
||||
with factory() as session:
|
||||
return {a.id: a for a in session.execute(select(Asset)).scalars()}
|
||||
|
||||
return _snapshot
|
||||
|
||||
|
||||
def test_initial_scan_registers_new_assets(tmp_path, inventory, make_image):
|
||||
lib = tmp_path / "lib"
|
||||
make_image(lib / "a.jpg", seed=1)
|
||||
make_image(lib / "album" / "b.jpg", seed=2)
|
||||
|
||||
result = inventory.scan(lib)
|
||||
|
||||
assert result.counts.get(Occurrence.NEW.value) == 2
|
||||
assert set(result.occurrences.values()) == {Occurrence.NEW.value}
|
||||
assert len(result.asset_ids) == 2
|
||||
|
||||
|
||||
def test_rescan_unchanged_is_idempotent(
|
||||
tmp_path, inventory, db_url, make_factory, make_image, assets_by_id
|
||||
):
|
||||
lib = tmp_path / "lib"
|
||||
make_image(lib / "a.jpg", seed=1)
|
||||
make_image(lib / "b.jpg", seed=2)
|
||||
|
||||
first = inventory.scan(lib)
|
||||
before = assets_by_id(make_factory(db_url))
|
||||
|
||||
second = inventory.scan(lib)
|
||||
after = assets_by_id(make_factory(db_url))
|
||||
|
||||
assert set(second.occurrences.values()) == {Occurrence.UNCHANGED.value}
|
||||
assert set(before) == set(after) # same asset ids, none created
|
||||
assert first.asset_ids == second.asset_ids
|
||||
|
||||
|
||||
def test_move_preserves_identity(tmp_path, inventory, db_url, make_factory, make_image):
|
||||
lib = tmp_path / "lib"
|
||||
src = make_image(lib / "a.jpg", seed=1)
|
||||
original_id = next(iter(inventory.scan(lib).asset_ids.values()))
|
||||
|
||||
dst = lib / "album" / "moved.jpg"
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(src), str(dst))
|
||||
|
||||
result = inventory.scan(lib)
|
||||
assert result.occurrences[str(dst)] == Occurrence.MOVED.value
|
||||
assert result.asset_ids[str(dst)] == original_id # id retained across the move
|
||||
|
||||
# Path history: old occurrence closed, new occurrence open.
|
||||
with make_factory(db_url)() as session:
|
||||
paths = session.query(AssetPath).filter_by(asset_id=original_id).all()
|
||||
by_path = {p.path: p for p in paths}
|
||||
assert by_path[str(src)].valid_until is not None
|
||||
assert by_path[str(dst)].valid_until is None
|
||||
|
||||
|
||||
def test_copy_creates_distinct_identity(tmp_path, inventory, make_image):
|
||||
lib = tmp_path / "lib"
|
||||
src = make_image(lib / "a.jpg", seed=1)
|
||||
first = inventory.scan(lib)
|
||||
original_id = first.asset_ids[str(src)]
|
||||
|
||||
copy = lib / "a_copy.jpg"
|
||||
shutil.copy2(str(src), str(copy))
|
||||
|
||||
result = inventory.scan(lib)
|
||||
assert result.occurrences[str(copy)] == Occurrence.COPIED.value
|
||||
assert result.asset_ids[str(copy)] != original_id
|
||||
assert result.asset_ids[str(src)] == original_id
|
||||
|
||||
|
||||
def test_replace_versus_metadata_change(tmp_path, inventory, make_image):
|
||||
lib = tmp_path / "lib"
|
||||
same = make_image(lib / "same.jpg", seed=1)
|
||||
other = make_image(lib / "other.jpg", seed=2)
|
||||
ids = inventory.scan(lib).asset_ids
|
||||
|
||||
# Metadata-style edit: append trailing bytes (pixels unchanged).
|
||||
with open(same, "ab") as handle:
|
||||
handle.write(b"\xff\xfe trailing")
|
||||
# Content replacement: overwrite with different pixels.
|
||||
make_image(other, seed=99)
|
||||
|
||||
result = inventory.scan(lib)
|
||||
assert result.occurrences[str(same)] == Occurrence.METADATA_CHANGED.value
|
||||
assert result.occurrences[str(other)] == Occurrence.REPLACED.value
|
||||
# Identity survives both; hashes refreshed.
|
||||
assert result.asset_ids[str(same)] == ids[str(same)]
|
||||
assert result.asset_ids[str(other)] == ids[str(other)]
|
||||
|
||||
|
||||
def test_missing_file_is_flagged_not_deleted(
|
||||
tmp_path, inventory, db_url, make_factory, make_image, assets_by_id
|
||||
):
|
||||
lib = tmp_path / "lib"
|
||||
a = make_image(lib / "a.jpg", seed=1)
|
||||
make_image(lib / "b.jpg", seed=2)
|
||||
ids = inventory.scan(lib).asset_ids
|
||||
missing_id = ids[str(a)]
|
||||
|
||||
a.unlink()
|
||||
result = inventory.scan(lib)
|
||||
|
||||
assert result.occurrences[str(a)] == Occurrence.MISSING.value
|
||||
assets = assets_by_id(make_factory(db_url))
|
||||
assert missing_id in assets # not pruned
|
||||
assert assets[missing_id].missing_at is not None
|
||||
assert assets[missing_id].availability_state == "active"
|
||||
|
||||
|
||||
def test_reappearing_file_clears_missing(
|
||||
tmp_path, inventory, db_url, make_factory, make_image, assets_by_id
|
||||
):
|
||||
lib = tmp_path / "lib"
|
||||
a = make_image(lib / "a.jpg", seed=1)
|
||||
ids = inventory.scan(lib).asset_ids
|
||||
asset_id = ids[str(a)]
|
||||
|
||||
backup = tmp_path / "backup.jpg"
|
||||
shutil.move(str(a), str(backup))
|
||||
inventory.scan(lib) # marks missing
|
||||
shutil.move(str(backup), str(a)) # reappears at the same path
|
||||
|
||||
inventory.scan(lib)
|
||||
assets = assets_by_id(make_factory(db_url))
|
||||
assert assets[asset_id].missing_at is None
|
||||
|
||||
|
||||
def test_identity_and_state_durable_across_restart(
|
||||
tmp_path, db_url, make_factory, make_image, assets_by_id
|
||||
):
|
||||
from photo_pipeline.services.inventory import InventoryService
|
||||
|
||||
lib = tmp_path / "lib"
|
||||
src = make_image(lib / "a.jpg", seed=1)
|
||||
|
||||
service_a = InventoryService(make_factory(db_url))
|
||||
original_id = service_a.scan(lib).asset_ids[str(src)]
|
||||
|
||||
# "Restart": a brand-new service/session factory against the same database.
|
||||
dst = lib / "renamed.jpg"
|
||||
shutil.move(str(src), str(dst))
|
||||
service_b = InventoryService(make_factory(db_url))
|
||||
result = service_b.scan(lib)
|
||||
|
||||
assert result.asset_ids[str(dst)] == original_id
|
||||
assert result.occurrences[str(dst)] == Occurrence.MOVED.value
|
||||
|
||||
assets = assets_by_id(make_factory(db_url))
|
||||
assert assets[original_id].current_path == str(dst)
|
||||
48
tests/unit/test_hashing.py
Normal file
48
tests/unit/test_hashing.py
Normal file
@@ -0,0 +1,48 @@
|
||||
"""Hashing: byte identity vs normalized-pixel identity."""
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
from photo_pipeline.services import hashing
|
||||
|
||||
|
||||
def _write(path, seed, size=(48, 32)):
|
||||
rng = np.random.default_rng(seed)
|
||||
arr = rng.integers(0, 256, (size[1], size[0], 3), dtype=np.uint8)
|
||||
Image.fromarray(arr).save(path, quality=95)
|
||||
|
||||
|
||||
def test_sha256_is_deterministic_and_distinct(tmp_path):
|
||||
a, b = tmp_path / "a.jpg", tmp_path / "b.jpg"
|
||||
_write(a, 1)
|
||||
_write(b, 2)
|
||||
assert hashing.sha256_file(a) == hashing.sha256_file(a)
|
||||
assert hashing.sha256_file(a) != hashing.sha256_file(b)
|
||||
|
||||
|
||||
def test_trailing_bytes_change_sha_but_not_pixels(tmp_path):
|
||||
"""Appending bytes after a JPEG (an EXIF-only-style edit) changes the file
|
||||
hash while decoded pixels — and the pixel hash — stay the same."""
|
||||
img = tmp_path / "a.jpg"
|
||||
_write(img, 1)
|
||||
sha_before = hashing.sha256_file(img)
|
||||
pix_before = hashing.pixel_sha256(img)
|
||||
|
||||
with open(img, "ab") as handle:
|
||||
handle.write(b"\xff\xfe trailing comment bytes")
|
||||
|
||||
assert hashing.sha256_file(img) != sha_before
|
||||
assert hashing.pixel_sha256(img) == pix_before
|
||||
|
||||
|
||||
def test_different_images_have_different_pixel_hashes(tmp_path):
|
||||
a, b = tmp_path / "a.jpg", tmp_path / "b.jpg"
|
||||
_write(a, 1)
|
||||
_write(b, 2)
|
||||
assert hashing.pixel_sha256(a) != hashing.pixel_sha256(b)
|
||||
|
||||
|
||||
def test_safe_pixel_sha256_returns_none_for_garbage(tmp_path):
|
||||
bad = tmp_path / "bad.jpg"
|
||||
bad.write_bytes(b"not an image")
|
||||
assert hashing.safe_pixel_sha256(bad) is None
|
||||
83
tests/unit/test_path_policy.py
Normal file
83
tests/unit/test_path_policy.py
Normal file
@@ -0,0 +1,83 @@
|
||||
"""Path policy: supported extensions, exclusion, deterministic discovery, and
|
||||
symlink-escape rejection."""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from photo_pipeline import path_policy
|
||||
|
||||
|
||||
def test_is_supported_is_case_insensitive():
|
||||
assert path_policy.is_supported("a.jpg")
|
||||
assert path_policy.is_supported("a.JPG")
|
||||
assert path_policy.is_supported("a.HEIC")
|
||||
assert not path_policy.is_supported("a.txt")
|
||||
assert not path_policy.is_supported("a.mp4")
|
||||
|
||||
|
||||
def test_is_excluded_matches_ignore_and_thumb_anywhere():
|
||||
assert path_policy.is_excluded("lib/_IGNORE/secret.jpg")
|
||||
assert path_policy.is_excluded("lib/album/.@__thumb/t.jpg")
|
||||
assert not path_policy.is_excluded("lib/album/photo.jpg")
|
||||
|
||||
|
||||
def test_discovery_finds_supported_and_skips_excluded_and_unsupported(tmp_path):
|
||||
(tmp_path / "album").mkdir()
|
||||
(tmp_path / "_IGNORE").mkdir()
|
||||
(tmp_path / "album" / ".@__thumb").mkdir()
|
||||
keep = [tmp_path / "root.jpg", tmp_path / "album" / "a.PNG"]
|
||||
for p in keep:
|
||||
p.write_bytes(b"x")
|
||||
(tmp_path / "notes.txt").write_bytes(b"x")
|
||||
(tmp_path / "_IGNORE" / "secret.jpg").write_bytes(b"x")
|
||||
(tmp_path / "album" / ".@__thumb" / "thumb.jpg").write_bytes(b"x")
|
||||
|
||||
found = path_policy.discover([tmp_path])
|
||||
assert found == sorted(keep)
|
||||
|
||||
|
||||
def test_discovery_is_deterministic(tmp_path):
|
||||
for i in range(5):
|
||||
(tmp_path / f"{i}.jpg").write_bytes(b"x")
|
||||
assert path_policy.discover([tmp_path]) == path_policy.discover([tmp_path])
|
||||
|
||||
|
||||
def test_resolve_within_accepts_inside_and_rejects_outside(tmp_path):
|
||||
root = tmp_path / "lib"
|
||||
root.mkdir()
|
||||
inside = root / "a.jpg"
|
||||
inside.write_bytes(b"x")
|
||||
assert path_policy.resolve_within(root, inside) == inside.resolve()
|
||||
with pytest.raises(path_policy.PathPolicyError):
|
||||
path_policy.resolve_within(root, tmp_path / "outside.jpg")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not hasattr(os, "symlink"), reason="symlinks unsupported")
|
||||
def test_symlink_escaping_root_fails_discovery(tmp_path):
|
||||
root = tmp_path / "lib"
|
||||
root.mkdir()
|
||||
outside = tmp_path / "outside.jpg"
|
||||
outside.write_bytes(b"x")
|
||||
link = root / "link.jpg"
|
||||
try:
|
||||
os.symlink(outside, link)
|
||||
except (OSError, NotImplementedError):
|
||||
pytest.skip("cannot create symlink on this platform")
|
||||
with pytest.raises(path_policy.PathPolicyError):
|
||||
path_policy.discover([root])
|
||||
|
||||
|
||||
@pytest.mark.skipif(not hasattr(os, "symlink"), reason="symlinks unsupported")
|
||||
def test_symlink_within_root_is_allowed(tmp_path):
|
||||
root = tmp_path / "lib"
|
||||
root.mkdir()
|
||||
target = root / "real.jpg"
|
||||
target.write_bytes(b"x")
|
||||
link = root / "alias.jpg"
|
||||
try:
|
||||
os.symlink(target, link)
|
||||
except (OSError, NotImplementedError):
|
||||
pytest.skip("cannot create symlink on this platform")
|
||||
found = path_policy.discover([root])
|
||||
assert target in found and link in found
|
||||
Reference in New Issue
Block a user