US01-04: Detect and Decide Duplicates (#49)

2026-07-15 17:14:24 +02:00
parent c25d45d8f7
commit 691d5a5651
9 changed files with 1137 additions and 4 deletions
--- a/migrations/versions/0002_duplicate_engine.py
+++ b/migrations/versions/0002_duplicate_engine.py
@@ -0,0 +1,99 @@
+"""Duplicate engine: canonical link, phash version, clusters, members, negative links.
+
+Revision ID: 0002_duplicate_engine
+Revises: 0001_initial_identity
+Create Date: 2026-07-15
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0002_duplicate_engine"
+down_revision = "0001_initial_identity"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # Plain ADD COLUMN: SQLite cannot ALTER-ADD a foreign key, and the ORM model
+    # keeps the ForeignKey for relationship metadata. canonical_asset_id is a soft
+    # self-link validated in the service (no cycles), not DB-enforced.
+    op.add_column("assets", sa.Column("phash_version", sa.Integer(), nullable=True))
+    op.add_column("assets", sa.Column("canonical_asset_id", sa.String(), nullable=True))
+    op.create_index("ix_assets_canonical_asset_id", "assets", ["canonical_asset_id"])
+
+    op.create_table(
+        "duplicate_clusters",
+        sa.Column("id", sa.String(), primary_key=True),
+        sa.Column("method", sa.String(), nullable=False),
+        sa.Column("confidence", sa.String(), nullable=False),
+        sa.Column("state", sa.String(), nullable=False),
+        sa.Column("decision", sa.String(), nullable=True),
+        sa.Column(
+            "canonical_asset_id", sa.String(), sa.ForeignKey("assets.id"), nullable=True
+        ),
+        sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+        sa.Column("decided_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.create_index("ix_duplicate_clusters_state", "duplicate_clusters", ["state"])
+
+    op.create_table(
+        "duplicate_members",
+        sa.Column(
+            "cluster_id",
+            sa.String(),
+            sa.ForeignKey("duplicate_clusters.id"),
+            primary_key=True,
+        ),
+        sa.Column(
+            "asset_id", sa.String(), sa.ForeignKey("assets.id"), primary_key=True
+        ),
+        sa.Column("role", sa.String(), nullable=False, server_default="member"),
+        sa.Column("distance", sa.Integer(), nullable=True),
+        sa.Column("evidence", sa.String(), nullable=True),
+        sa.Column(
+            "added_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+    )
+    op.create_index("ix_duplicate_members_asset_id", "duplicate_members", ["asset_id"])
+
+    op.create_table(
+        "duplicate_negative_links",
+        sa.Column(
+            "asset_a", sa.String(), sa.ForeignKey("assets.id"), primary_key=True
+        ),
+        sa.Column(
+            "asset_b", sa.String(), sa.ForeignKey("assets.id"), primary_key=True
+        ),
+        sa.Column("reason", sa.String(), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("duplicate_negative_links")
+    op.drop_table("duplicate_members")
+    op.drop_table("duplicate_clusters")
+    op.drop_index("ix_assets_canonical_asset_id", "assets")
+    op.drop_column("assets", "canonical_asset_id")
+    op.drop_column("assets", "phash_version")