US01-04: Detect and Decide Duplicates (#49)

tests/integration/test_duplicate_engine.py

@@ -0,0 +1,291 @@
+"""Duplicate engine: banded clustering, reversible decisions, negative links,
+reopening on contradiction, no canonical cycles, and durability across restart.
+
+Fixtures are inlined (not in a sibling conftest.py) to avoid shadowing the
+characterization suite's bare ``conftest`` import under pytest's prepend mode.
+"""
+
+import shutil
+
+import numpy as np
+import pytest
+from PIL import Image
+from sqlalchemy import select
+
+from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
+from photo_pipeline.models import Asset, DuplicateNegativeLink
+from photo_pipeline.services.duplicates import (
+    ClusterState,
+    ConflictError,
+    Decision,
+    DuplicateError,
+    DuplicateService,
+    Method,
+)
+from photo_pipeline.services.inventory import InventoryService
+
+
+@pytest.fixture
+def db_url(tmp_path):
+    url = f"sqlite:///{tmp_path / 'dup.db'}"
+    run_migrations(url)
+    return url
+
+
+@pytest.fixture
+def make_factory():
+    engines = []
+
+    def _make(url):
+        engine = create_db_engine(url)
+        engines.append(engine)
+        return create_session_factory(engine)
+
+    yield _make
+    for engine in engines:
+        engine.dispose()
+
+
+@pytest.fixture
+def factory(db_url, make_factory):
+    return make_factory(db_url)
+
+
+@pytest.fixture
+def inventory(factory):
+    return InventoryService(factory)
+
+
+@pytest.fixture
+def duplicates(factory):
+    return DuplicateService(factory)
+
+
+def structured(path, seed, size=(256, 192)):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    rng = np.random.default_rng(seed)
+    w, h = size
+    base = np.zeros((h, w, 3), dtype=np.uint8)
+    for _ in range(6):
+        x0 = int(rng.integers(0, w - 60))
+        y0 = int(rng.integers(0, h - 60))
+        base[y0 : y0 + 60, x0 : x0 + 60] = rng.integers(0, 256, 3)
+    grad = np.linspace(0, 120, w, dtype=np.uint8)
+    base[:, :, 0] = np.clip(base[:, :, 0].astype(int) + grad[None, :], 0, 255)
+    Image.fromarray(base).save(path, quality=95)
+    return path
+
+
+def resized_copy(src, dst, scale=0.5):
+    with Image.open(src) as image:
+        image.resize(
+            (int(image.width * scale), int(image.height * scale)), Image.LANCZOS
+        ).save(dst, quality=95)
+    return dst
+
+
+def only(clusters, method):
+    return [c for c in clusters if c["method"] == method]
+
+
+def test_exact_copies_form_auto_decided_cluster(tmp_path, inventory, duplicates, factory):
+    lib = tmp_path / "lib"
+    a = structured(lib / "a.jpg", 1)
+    shutil.copy2(a, lib / "a_copy.jpg")
+    inventory.scan(lib)
+
+    clusters = duplicates.detect().clusters
+    exact = only(clusters, Method.EXACT.value)
+    assert len(exact) == 1
+    cluster = exact[0]
+    assert cluster["state"] == ClusterState.DECIDED.value
+    assert cluster["decision"] == Decision.CANONICAL.value
+    roles = {m["role"] for m in cluster["members"]}
+    assert roles == {"canonical", "variant"}
+
+    # The variant asset points at the canonical; the canonical points at nothing.
+    assets = {a.id: a for a in factory().execute(select(Asset)).scalars()}
+    canonical = cluster["canonical_asset_id"]
+    for m in cluster["members"]:
+        if m["role"] == "variant":
+            assert assets[m["asset_id"]].canonical_asset_id == canonical
+        else:
+            assert assets[m["asset_id"]].canonical_asset_id is None
+
+
+def test_same_pixels_different_bytes_form_pixel_cluster(tmp_path, inventory, duplicates):
+    lib = tmp_path / "lib"
+    a = structured(lib / "a.jpg", 1)
+    twin = lib / "a_meta.jpg"
+    shutil.copy2(a, twin)
+    with open(twin, "ab") as handle:  # trailing bytes: same pixels, different bytes
+        handle.write(b"\xff\xfe metadata")
+    inventory.scan(lib)
+
+    clusters = duplicates.detect().clusters
+    pixel = only(clusters, Method.PIXEL.value)
+    assert len(pixel) == 1
+    assert pixel[0]["confidence"] == "pixel"
+    assert pixel[0]["state"] == ClusterState.DECIDED.value
+
+
+def test_perceptual_variant_is_review_only(tmp_path, inventory, duplicates):
+    lib = tmp_path / "lib"
+    a = structured(lib / "a.jpg", 1)
+    resized_copy(a, lib / "a_small.jpg")
+    inventory.scan(lib)
+
+    clusters = duplicates.detect().clusters
+    perceptual = only(clusters, Method.PERCEPTUAL.value)
+    assert len(perceptual) == 1
+    assert perceptual[0]["state"] == ClusterState.OPEN.value  # never auto-decided
+    assert perceptual[0]["decision"] is None
+    assert perceptual[0]["confidence"] in ("near", "similar")
+
+
+def test_distinct_images_do_not_cluster(tmp_path, inventory, duplicates):
+    lib = tmp_path / "lib"
+    structured(lib / "a.jpg", 1)
+    structured(lib / "b.jpg", 2)
+    inventory.scan(lib)
+    assert duplicates.detect().clusters == []
+
+
+def test_not_duplicate_decision_persists_negative_link_and_suppresses(
+    tmp_path, inventory, duplicates, factory
+):
+    lib = tmp_path / "lib"
+    a = structured(lib / "a.jpg", 1)
+    resized_copy(a, lib / "a_small.jpg")
+    inventory.scan(lib)
+    cluster = only(duplicates.detect().clusters, Method.PERCEPTUAL.value)[0]
+
+    duplicates.decide(cluster["id"], Decision.NOT_DUPLICATE)
+
+    links = list(factory().execute(select(DuplicateNegativeLink)).scalars())
+    assert len(links) == 1
+
+    # Re-detecting must not re-suggest the rejected pair.
+    again = only(duplicates.detect().clusters, Method.PERCEPTUAL.value)
+    assert again == []
+
+
+def test_decision_is_reversible(tmp_path, inventory, duplicates, factory):
+    lib = tmp_path / "lib"
+    a = structured(lib / "a.jpg", 1)
+    resized_copy(a, lib / "a_small.jpg")
+    inventory.scan(lib)
+    cluster = only(duplicates.detect().clusters, Method.PERCEPTUAL.value)[0]
+    members = [m["asset_id"] for m in cluster["members"]]
+
+    duplicates.decide(cluster["id"], Decision.NOT_DUPLICATE)
+    assert factory().execute(select(DuplicateNegativeLink)).scalars().all()
+
+    # Reverse: choose a canonical instead.
+    reversed_snapshot = duplicates.decide(
+        cluster["id"], Decision.CANONICAL, canonical_asset_id=members[0]
+    )
+    assert reversed_snapshot["state"] == ClusterState.DECIDED.value
+    assert not factory().execute(select(DuplicateNegativeLink)).scalars().all()
+
+
+def test_version_conflict_is_rejected(tmp_path, inventory, duplicates):
+    lib = tmp_path / "lib"
+    a = structured(lib / "a.jpg", 1)
+    resized_copy(a, lib / "a_small.jpg")
+    inventory.scan(lib)
+    cluster = only(duplicates.detect().clusters, Method.PERCEPTUAL.value)[0]
+
+    with pytest.raises(ConflictError):
+        duplicates.decide(
+            cluster["id"], Decision.DEFERRED, expected_version=cluster["version"] + 5
+        )
+
+
+def test_canonical_must_be_a_member(tmp_path, inventory, duplicates):
+    lib = tmp_path / "lib"
+    a = structured(lib / "a.jpg", 1)
+    resized_copy(a, lib / "a_small.jpg")
+    inventory.scan(lib)
+    cluster = only(duplicates.detect().clusters, Method.PERCEPTUAL.value)[0]
+
+    with pytest.raises(DuplicateError):
+        duplicates.decide(cluster["id"], Decision.CANONICAL, canonical_asset_id="ghost")
+
+
+def test_no_canonical_cycles(tmp_path, inventory, duplicates, factory):
+    lib = tmp_path / "lib"
+    a = structured(lib / "a.jpg", 1)
+    shutil.copy2(a, lib / "a_copy.jpg")
+    inventory.scan(lib)
+    duplicates.detect()
+
+    # Every asset's canonical chain terminates (no loop).
+    assets = {a.id: a for a in factory().execute(select(Asset)).scalars()}
+    for start in assets:
+        seen, cur = set(), start
+        while cur is not None:
+            assert cur not in seen
+            seen.add(cur)
+            cur = assets[cur].canonical_asset_id
+
+
+def test_new_exact_member_inherits_decision(tmp_path, inventory, duplicates, factory):
+    lib = tmp_path / "lib"
+    a = structured(lib / "a.jpg", 1)
+    shutil.copy2(a, lib / "a2.jpg")
+    inventory.scan(lib)
+    cluster = only(duplicates.detect().clusters, Method.EXACT.value)[0]
+    canonical = cluster["canonical_asset_id"]
+
+    # A third exact copy appears later.
+    shutil.copy2(a, lib / "a3.jpg")
+    inventory.scan(lib)
+    cluster2 = only(duplicates.detect().clusters, Method.EXACT.value)[0]
+
+    assert cluster2["id"] == cluster["id"]
+    assert cluster2["state"] == ClusterState.DECIDED.value
+    assert len(cluster2["members"]) == 3
+    assets = {a.id: a for a in factory().execute(select(Asset)).scalars()}
+    variants = [m for m in cluster2["members"] if m["role"] == "variant"]
+    assert len(variants) == 2
+    for m in variants:
+        assert assets[m["asset_id"]].canonical_asset_id == canonical
+
+
+def test_contradictory_member_reopens_reviewed_cluster(
+    tmp_path, inventory, duplicates, db_url, make_factory
+):
+    lib = tmp_path / "lib"
+    a = structured(lib / "a.jpg", 1)
+    b = resized_copy(a, lib / "b.jpg")
+    inventory.scan(lib)
+    cluster = only(duplicates.detect().clusters, Method.PERCEPTUAL.value)[0]
+    duplicates.decide(cluster["id"], Decision.NOT_DUPLICATE)  # dismissed + negative link
+
+    # b is replaced by an exact copy of a — now byte-identical to a "not duplicate".
+    shutil.copy2(a, b)
+    inventory.scan(lib)
+    clusters = duplicates.detect().clusters
+
+    reopened = [c for c in clusters if c["state"] == ClusterState.REOPENED.value]
+    assert reopened, "byte-identical contradiction of a not_duplicate must reopen review"
+
+
+def test_decisions_durable_across_restart(tmp_path, db_url, make_factory):
+    lib = tmp_path / "lib"
+    a = structured(lib / "a.jpg", 1)
+    shutil.copy2(a, lib / "a_copy.jpg")
+
+    InventoryService(make_factory(db_url)).scan(lib)
+    cluster = only(
+        DuplicateService(make_factory(db_url)).detect().clusters, Method.EXACT.value
+    )[0]
+
+    # Fresh factory against the same database: the decision persisted.
+    from photo_pipeline.models import DuplicateCluster
+
+    with make_factory(db_url)() as session:
+        persisted = session.get(DuplicateCluster, cluster["id"])
+        assert persisted.state == ClusterState.DECIDED.value
+        assert persisted.decision == Decision.CANONICAL.value

tests/integration/test_migrations.py

@@ -7,6 +7,16 @@ import pytest
 
 from photo_pipeline.db import run_migrations
 
+
+def _alembic_head() -> str:
+    from alembic.config import Config as AlembicConfig
+    from alembic.script import ScriptDirectory
+
+    repo = Path(__file__).resolve().parents[2]
+    cfg = AlembicConfig(str(repo / "alembic.ini"))
+    cfg.set_main_option("script_location", str(repo / "migrations"))
+    return ScriptDirectory.from_config(cfg).get_current_head()
+
 # Faithful snapshot of the donor photo_analyzer.py schema (photos + FTS + triggers).
 LEGACY_SCHEMA = """
 CREATE TABLE photos (
@@ -50,7 +60,7 @@ def test_migrations_from_empty_database(tmp_path):
 
     tables = _tables(db)
     assert {"assets", "asset_paths"} <= tables
-    assert _head_revision(db) == "0001_initial_identity"
+    assert _head_revision(db) == _alembic_head()
 
 
 def test_migrations_from_legacy_snapshot_preserve_existing_data(tmp_path):
@@ -75,7 +85,7 @@ def test_migrations_from_legacy_snapshot_preserve_existing_data(tmp_path):
     finally:
         conn.close()
     assert row == ("album/one.jpg", "analyzed", "a red block")
-    assert _head_revision(db) == "0001_initial_identity"
+    assert _head_revision(db) == _alembic_head()
 
 
 def test_migrations_are_idempotent(tmp_path):
@@ -83,7 +93,7 @@ def test_migrations_are_idempotent(tmp_path):
     url = f"sqlite:///{db}"
     run_migrations(url)
     run_migrations(url)  # second run is a no-op at head
-    assert _head_revision(db) == "0001_initial_identity"
+    assert _head_revision(db) == _alembic_head()
 
 
 @pytest.mark.parametrize("expected", ["assets", "asset_paths"])