325 lines
12 KiB
Python
325 lines
12 KiB
Python
"""Performance budgets, paging, and resource bounds (US07-06).
|
|
|
|
The harness itself is the deliverable, so this suite proves the harness: that it
|
|
builds a synthetic library, measures the same scenarios every time, exports
|
|
machine-readable metrics, and — the part that matters — *fails* when a budget is
|
|
exceeded rather than printing a number nobody reads.
|
|
|
|
It runs the ``smoke`` profile. The 25k/100k/500k matrix is a scheduled command
|
|
(README "Performance budgets"), because minutes of build time do not belong in the
|
|
suite that runs on every change.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
from sqlalchemy import func, select
|
|
|
|
from photo_pipeline.api.app import create_app
|
|
from photo_pipeline.config import Config
|
|
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
|
|
from photo_pipeline.models import Asset, DuplicateCluster, DuplicateMember, SafetyReview
|
|
from photo_pipeline.services import benchmarks
|
|
from photo_pipeline.services.duplicates import MAX_MEMBER_PAGE, DuplicateService
|
|
from photo_pipeline.services.safety import SafetyService
|
|
|
|
NOW = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
|
|
|
|
|
def _config(tmp_path, **extra) -> Config:
|
|
data = tmp_path / "data"
|
|
data.mkdir(parents=True, exist_ok=True)
|
|
lib = tmp_path / "lib"
|
|
lib.mkdir(exist_ok=True)
|
|
return Config.from_env(
|
|
{
|
|
"PHOTO_PIPELINE_DATA_DIR": str(data),
|
|
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(lib),
|
|
**extra,
|
|
}
|
|
)
|
|
|
|
|
|
# ── the harness ──────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_the_smoke_profile_measures_every_scenario_and_passes_its_budgets(tmp_path):
|
|
config = _config(tmp_path)
|
|
output = tmp_path / "report.json"
|
|
|
|
report = benchmarks.run(config, profile="smoke", output=output)
|
|
|
|
assert report["ok"] is True, report["breaches"]
|
|
assert json.loads(output.read_text())["profile"] == "smoke" # machine-readable
|
|
run = report["runs"][0]
|
|
measured = {scenario["scenario"] for scenario in run["scenarios"]}
|
|
assert measured == {
|
|
"inventory_page",
|
|
"library_search",
|
|
"library_stats",
|
|
"workflow_readiness",
|
|
"duplicate_cluster_list",
|
|
"duplicate_cluster_page",
|
|
}
|
|
for scenario in run["scenarios"]:
|
|
assert scenario["latency_p95_ms"] >= scenario["latency_p50_ms"]
|
|
assert scenario["iterations"] == benchmarks.PROFILES["smoke"]["iterations"]
|
|
for metric in ("rss_bytes", "open_files", "db_bytes", "wal_bytes", "queue_depth"):
|
|
assert metric in run["resources"]
|
|
assert run["build"]["assets"] == 2_000
|
|
|
|
|
|
def test_a_breached_budget_fails_the_run_and_names_what_broke(tmp_path, monkeypatch):
|
|
config = _config(tmp_path)
|
|
# A budget nothing can meet: the run must fail, not shrug.
|
|
monkeypatch.setattr(
|
|
benchmarks,
|
|
"BUDGETS",
|
|
(benchmarks.Budget("latency_p95_ms", 0.0000001, "ms", "deliberately impossible"),),
|
|
)
|
|
|
|
report = benchmarks.run(config, profile="smoke")
|
|
|
|
assert report["ok"] is False
|
|
breach = report["breaches"][0]
|
|
assert {"scope", "metric", "value", "limit", "unit", "why"} <= set(breach)
|
|
assert breach["metric"] == "latency_p95_ms" and breach["value"] > breach["limit"]
|
|
|
|
|
|
def test_an_approved_exception_raises_the_limit_and_is_recorded(tmp_path, monkeypatch):
|
|
config = _config(tmp_path)
|
|
monkeypatch.setattr(
|
|
benchmarks,
|
|
"BUDGETS",
|
|
(benchmarks.Budget("latency_p95_ms", 0.0000001, "ms", "deliberately impossible"),),
|
|
)
|
|
monkeypatch.setattr(
|
|
benchmarks,
|
|
"APPROVED_EXCEPTIONS",
|
|
{
|
|
("smoke", "library_stats", "latency_p95_ms"): {
|
|
"limit": 10_000,
|
|
"approved_by": "test",
|
|
"reason": "documenting the mechanism",
|
|
"review_by": "2026-12-31",
|
|
}
|
|
},
|
|
)
|
|
|
|
report = benchmarks.run(config, profile="smoke")
|
|
|
|
scopes = {breach["scope"] for breach in report["breaches"]}
|
|
assert "library_stats" not in scopes, "the approved exception was not applied"
|
|
assert report["exceptions_applied"][0]["approved_by"] == "test"
|
|
|
|
|
|
def test_an_unknown_profile_is_refused(tmp_path):
|
|
with pytest.raises(ValueError, match="unknown profile"):
|
|
benchmarks.run(_config(tmp_path), profile="enormous")
|
|
|
|
|
|
def test_the_soak_reports_growth_queue_depth_and_leaves_no_backlog(tmp_path):
|
|
config = _config(tmp_path)
|
|
benchmarks.synthesize(config, assets=200, cluster_members=0)
|
|
engine = create_db_engine(config.database_url)
|
|
factory = create_session_factory(engine)
|
|
try:
|
|
result = benchmarks.soak(config, factory, seconds=1.5, interval=0.25)
|
|
finally:
|
|
engine.dispose()
|
|
|
|
assert result["cycles"] > 0 and len(result["samples"]) >= 2
|
|
# Every cycle enqueues and cancels a job: the lane must end empty, which is the
|
|
# difference between "busy" and "growing without bound".
|
|
assert result["queue_depth"] == 0
|
|
assert result["rss_growth_bytes"] >= 0
|
|
assert result["open_files"] <= 256
|
|
|
|
|
|
# ── paging large clusters ────────────────────────────────────────────────────
|
|
|
|
|
|
def _cluster(config: Config, members: int) -> tuple[str, object]:
|
|
run_migrations(config.database_url)
|
|
engine = create_db_engine(config.database_url)
|
|
factory = create_session_factory(engine)
|
|
cluster_id = str(uuid.uuid4())
|
|
with factory() as session:
|
|
session.add(
|
|
DuplicateCluster(
|
|
id=cluster_id, method="perceptual", confidence="near", state="open", version=1
|
|
)
|
|
)
|
|
session.flush()
|
|
for index in range(members):
|
|
asset_id = f"member-{index:06d}"
|
|
session.add(
|
|
Asset(
|
|
id=asset_id,
|
|
original_path=f"/lib/photo-{index}.jpg",
|
|
current_path=f"/lib/photo-{index}.jpg",
|
|
discovered_at=NOW,
|
|
hash_version=1,
|
|
byte_size=1024,
|
|
)
|
|
)
|
|
session.add(
|
|
DuplicateMember(cluster_id=cluster_id, asset_id=asset_id, role="member", distance=1)
|
|
)
|
|
session.commit()
|
|
return cluster_id, factory
|
|
|
|
|
|
def test_a_cluster_of_thousands_is_paged_not_dumped(tmp_path):
|
|
config = _config(tmp_path)
|
|
cluster_id, factory = _cluster(config, 3_000)
|
|
service = DuplicateService(factory)
|
|
|
|
first = service.get_cluster(cluster_id)
|
|
|
|
assert first["member_total"] == 3_000
|
|
assert len(first["members"]) == 100, "the default page, not the whole cluster"
|
|
second = service.get_cluster(cluster_id, limit=100, offset=100)
|
|
assert [m["asset_id"] for m in second["members"]][0] == "member-000100"
|
|
assert not {m["asset_id"] for m in first["members"]} & {
|
|
m["asset_id"] for m in second["members"]
|
|
}
|
|
# The last page is short and the pages together cover the cluster exactly.
|
|
tail = service.get_cluster(cluster_id, limit=MAX_MEMBER_PAGE, offset=2_900)
|
|
assert len(tail["members"]) == 100
|
|
|
|
|
|
def test_the_cluster_list_carries_counts_without_loading_every_member(tmp_path):
|
|
config = _config(tmp_path)
|
|
cluster_id, factory = _cluster(config, 3_000)
|
|
|
|
listed = DuplicateService(factory).list_clusters(limit=50)
|
|
|
|
entry = listed["items"][0]
|
|
assert entry["id"] == cluster_id
|
|
assert entry["member_total"] == 3_000
|
|
assert len(entry["members"]) <= 20, "the list view shows a preview, never the cluster"
|
|
|
|
|
|
def test_the_api_pages_cluster_members_and_bounds_the_page_size(tmp_path):
|
|
config = _config(tmp_path)
|
|
cluster_id, _ = _cluster(config, 1_200)
|
|
with TestClient(create_app(config)) as client:
|
|
default = client.get(f"/api/v1/duplicates/clusters/{cluster_id}").json()
|
|
assert default["member_total"] == 1_200 and len(default["members"]) == 100
|
|
|
|
paged = client.get(
|
|
f"/api/v1/duplicates/clusters/{cluster_id}", params={"limit": 250, "offset": 1_000}
|
|
).json()
|
|
assert len(paged["members"]) == 200 and paged["offset"] == 1_000
|
|
|
|
# A caller cannot ask for the whole cluster by asking for a huge page.
|
|
assert (
|
|
client.get(
|
|
f"/api/v1/duplicates/clusters/{cluster_id}",
|
|
params={"limit": MAX_MEMBER_PAGE + 1},
|
|
).status_code
|
|
== 422
|
|
)
|
|
|
|
|
|
# ── the queries behind the pages ─────────────────────────────────────────────
|
|
|
|
|
|
def test_the_review_queue_is_filtered_and_paged_in_the_database(tmp_path):
|
|
"""A queue that loads every asset to slice 100 of them is the shape this story
|
|
exists to remove; the totals must stay exact while it pages."""
|
|
config = _config(tmp_path)
|
|
run_migrations(config.database_url)
|
|
engine = create_db_engine(config.database_url)
|
|
factory = create_session_factory(engine)
|
|
with factory() as session:
|
|
for index in range(500):
|
|
asset_id = f"asset-{index:04d}"
|
|
session.add(
|
|
Asset(
|
|
id=asset_id,
|
|
original_path=f"/lib/{index:04d}.jpg",
|
|
current_path=f"/lib/{index:04d}.jpg",
|
|
discovered_at=NOW,
|
|
hash_version=1,
|
|
byte_size=1,
|
|
)
|
|
)
|
|
if index % 2 == 0:
|
|
session.add(
|
|
SafetyReview(
|
|
id=str(uuid.uuid4()),
|
|
asset_id=asset_id,
|
|
decision="sfw",
|
|
created_at=NOW,
|
|
)
|
|
)
|
|
session.commit()
|
|
service = SafetyService(factory)
|
|
|
|
page = service.review_queue(state="undecided", limit=10, offset=0)
|
|
assert page["total"] == 250 and len(page["items"]) == 10
|
|
assert all(item["decision"] is None for item in page["items"])
|
|
|
|
second = service.review_queue(state="undecided", limit=10, offset=10)
|
|
assert not {item["asset_id"] for item in page["items"]} & {
|
|
item["asset_id"] for item in second["items"]
|
|
}
|
|
assert service.counts() == {
|
|
"sfw": 250,
|
|
"nsfw": 0,
|
|
"deferred": 0,
|
|
"undecided": 250,
|
|
"scored": 0,
|
|
}
|
|
engine.dispose()
|
|
|
|
|
|
def test_the_latest_review_still_wins_after_a_revision(tmp_path):
|
|
"""The counts are aggregated in SQL now; the rule they aggregate is unchanged."""
|
|
config = _config(tmp_path)
|
|
run_migrations(config.database_url)
|
|
engine = create_db_engine(config.database_url)
|
|
factory = create_session_factory(engine)
|
|
with factory() as session:
|
|
session.add(
|
|
Asset(
|
|
id="a",
|
|
original_path="/lib/a.jpg",
|
|
current_path="/lib/a.jpg",
|
|
discovered_at=NOW,
|
|
hash_version=1,
|
|
byte_size=1,
|
|
)
|
|
)
|
|
session.add(
|
|
SafetyReview(id="r1", asset_id="a", decision="sfw", score=0.1, created_at=NOW)
|
|
)
|
|
session.commit()
|
|
service = SafetyService(factory)
|
|
assert service.current_decision("a") == "sfw"
|
|
|
|
with factory() as session:
|
|
session.add(
|
|
SafetyReview(
|
|
id="r2",
|
|
asset_id="a",
|
|
decision="nsfw",
|
|
prior_decision="sfw",
|
|
created_at=NOW.replace(hour=2),
|
|
)
|
|
)
|
|
session.commit()
|
|
|
|
assert service.current_decision("a") == "nsfw"
|
|
assert service.counts()["nsfw"] == 1 and service.counts()["sfw"] == 0
|
|
with factory() as session: # the history itself is never rewritten
|
|
assert session.scalar(select(func.count()).select_from(SafetyReview)) == 2
|
|
engine.dispose()
|