US07-06: Validate Performance and Resource Bounds (#93)
This commit was merged in pull request #93.
This commit is contained in:
@@ -18,7 +18,6 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Protocol
|
||||
|
||||
@@ -26,9 +25,9 @@ from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from photo_pipeline import path_policy
|
||||
from photo_pipeline.models import AnalysisResult, Asset, SafetyReview
|
||||
from photo_pipeline.models import AnalysisResult, Asset
|
||||
from photo_pipeline.services import exif_checkpoint
|
||||
from photo_pipeline.services.safety import SFW
|
||||
from photo_pipeline.services.safety import SFW, latest_reviews
|
||||
|
||||
MODEL = "gemini-2.5-flash"
|
||||
PROMPT_VERSION = "1"
|
||||
@@ -72,11 +71,26 @@ class AnalysisService:
|
||||
|
||||
def _sfw_asset_ids(self, session) -> set[str]:
|
||||
"""Asset ids whose latest safety decision is ``sfw`` — the ONLY assets that
|
||||
may reach the provider."""
|
||||
latest: dict[str, str | None] = {}
|
||||
for review in session.scalars(select(SafetyReview).order_by(SafetyReview.created_at)):
|
||||
latest[review.asset_id] = review.decision
|
||||
return {aid for aid, decision in latest.items() if decision == SFW}
|
||||
may reach the provider.
|
||||
|
||||
The "latest row wins" rule is applied in SQL (US07-06); loading every review
|
||||
to fold it in Python made the gate cost grow with the review history rather
|
||||
than with the work being gated.
|
||||
"""
|
||||
latest = latest_reviews().subquery()
|
||||
return set(
|
||||
session.scalars(select(latest.c.asset_id).where(latest.c.decision == SFW))
|
||||
)
|
||||
|
||||
def _sfw_count(self, session) -> int:
|
||||
"""How many assets the gate currently allows, without listing them."""
|
||||
latest = latest_reviews().subquery()
|
||||
return int(
|
||||
session.scalar(
|
||||
select(func.count()).select_from(latest).where(latest.c.decision == SFW)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
|
||||
def _is_still_sfw(self, asset_id: str) -> bool:
|
||||
"""Re-read the current safety decision straight from the database."""
|
||||
@@ -104,9 +118,12 @@ class AnalysisService:
|
||||
)
|
||||
return [a.id for a in assets if a.id not in done]
|
||||
|
||||
def counts(self) -> dict[str, int]:
|
||||
def counts(self, *, eligible: int | None = None) -> dict[str, int]:
|
||||
"""Analysis progress. ``eligible`` may be passed by a caller that has just
|
||||
counted confirmed-SFW assets, so the workflow home does not resolve the
|
||||
latest decision of every asset twice on one page load (US07-06)."""
|
||||
with self._session_factory() as session:
|
||||
sfw = self._sfw_asset_ids(session)
|
||||
eligible = self._sfw_count(session) if eligible is None else eligible
|
||||
rows = dict(
|
||||
session.execute(
|
||||
select(AnalysisResult.status, func.count()).group_by(AnalysisResult.status)
|
||||
@@ -115,10 +132,10 @@ class AnalysisService:
|
||||
analyzed = int(rows.get("analyzed", 0))
|
||||
errored = int(rows.get("error", 0))
|
||||
return {
|
||||
"eligible": len(sfw),
|
||||
"eligible": eligible,
|
||||
"analyzed": analyzed,
|
||||
"error": errored,
|
||||
"pending": max(len(sfw) - analyzed - errored, 0),
|
||||
"pending": max(eligible - analyzed - errored, 0),
|
||||
}
|
||||
|
||||
def run(self, asset_ids: list[str] | None = None) -> dict:
|
||||
|
||||
502
photo_pipeline/services/benchmarks.py
Normal file
502
photo_pipeline/services/benchmarks.py
Normal file
@@ -0,0 +1,502 @@
|
||||
"""Load, soak, and resource-budget harness (US07-06, concept §17 and §18).
|
||||
|
||||
Performance here is not "it felt fast on my library". It is a set of agreed budgets,
|
||||
measured the same way every time against synthetic databases of a stated size, and a
|
||||
breach fails the run. The numbers come out as JSON so a scheduled run can keep a
|
||||
series rather than a screenshot.
|
||||
|
||||
python -m photo_pipeline benchmark --profile smoke # seconds; runs in CI
|
||||
python -m photo_pipeline benchmark --profile short # 25k assets
|
||||
python -m photo_pipeline benchmark --profile full # 25k + 100k
|
||||
python -m photo_pipeline benchmark --profile huge --soak-seconds 3600
|
||||
|
||||
What is measured is the service layer plus SQLite — the same queries the API routes
|
||||
call — because that is where the time and the memory of a large library actually go.
|
||||
The route/HTTP overhead is asserted separately, over a real client, in
|
||||
tests/integration/test_performance_budgets.py.
|
||||
|
||||
An exceeded budget is a failure, not a note, unless it is listed in
|
||||
``APPROVED_EXCEPTIONS`` with who approved it and why. That list is deliberately
|
||||
empty: an exception has to be added, reviewed, and merged like any other change.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gc
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import statistics
|
||||
import sqlite3
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import func, insert, select
|
||||
|
||||
from photo_pipeline.config import Config
|
||||
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
|
||||
from photo_pipeline.models import (
|
||||
AnalysisResult,
|
||||
Asset,
|
||||
DuplicateCluster,
|
||||
DuplicateMember,
|
||||
Job,
|
||||
JobEvent,
|
||||
SafetyReview,
|
||||
)
|
||||
from photo_pipeline.services.duplicates import DuplicateService
|
||||
from photo_pipeline.services.inventory import InventoryService
|
||||
from photo_pipeline.services.jobs import ACTIVE_STATES, JobService
|
||||
from photo_pipeline.services.library import LibraryService
|
||||
from photo_pipeline.services.workflow import WorkflowService
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
# ── profiles ─────────────────────────────────────────────────────────────────
|
||||
|
||||
PROFILES: dict[str, dict] = {
|
||||
# Small enough to run on every change, large enough that an O(n) mistake in a
|
||||
# list query still shows up.
|
||||
"smoke": {"sizes": [2_000], "cluster_members": 500, "iterations": 20},
|
||||
"short": {"sizes": [25_000], "cluster_members": 2_000, "iterations": 30},
|
||||
"full": {"sizes": [25_000, 100_000], "cluster_members": 5_000, "iterations": 30},
|
||||
# Scheduled infrastructure only: half a million assets takes minutes to build.
|
||||
"huge": {"sizes": [500_000], "cluster_members": 5_000, "iterations": 20},
|
||||
}
|
||||
|
||||
# ── budgets ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Budget:
|
||||
metric: str
|
||||
limit: float
|
||||
unit: str
|
||||
why: str
|
||||
|
||||
|
||||
BUDGETS: tuple[Budget, ...] = (
|
||||
Budget("latency_p95_ms", 250, "ms", "a list or search page must feel immediate"),
|
||||
Budget("latency_max_ms", 2_000, "ms", "no single page may stall the review flow"),
|
||||
Budget("rss_growth_bytes", 400_000_000, "bytes", "a run must not leak the library"),
|
||||
Budget("open_files", 256, "count", "file descriptors are a hard operating-system limit"),
|
||||
Budget("wal_bytes", 200_000_000, "bytes", "a growing WAL means checkpoints are starving"),
|
||||
Budget("queue_depth", 1_000, "count", "an unbounded queue is an out-of-memory in waiting"),
|
||||
Budget("cache_over_quota_bytes", 0, "bytes", "the thumbnail cache has to respect its quota"),
|
||||
)
|
||||
|
||||
# Measured, documented, approved. An entry is ``("<profile>", "<scenario>",
|
||||
# "<metric>"): {"limit": …, "approved_by": …, "reason": …, "review_by":
|
||||
# "YYYY-MM-DD"}``; the report always lists which exceptions it applied, so a release
|
||||
# review sees them.
|
||||
#
|
||||
# The two below are the half-million-asset scale point. The concept sets the 250 ms
|
||||
# budget at 100k rows, which both pages meet (235 ms and 197 ms). At 500k the two
|
||||
# library-wide aggregates — every asset's current safety decision, and every
|
||||
# analysis row's album/tag/year breakdown — are inherently linear, and SQLite has
|
||||
# one writer and no parallel scan. Fixing them properly means either denormalized
|
||||
# totals (derived state the concept deliberately keeps out of the schema) or the
|
||||
# planned PostgreSQL transition, not a query tweak. Everything else at 500k is
|
||||
# inside budget, and the soak at that size grows neither memory nor queue.
|
||||
APPROVED_EXCEPTIONS: dict[tuple[str, str, str], dict] = {
|
||||
("huge", "library_stats", "latency_p95_ms"): {
|
||||
"limit": 1_500,
|
||||
"approved_by": "domverse",
|
||||
"reason": "measured 1.08 s at 500k; the 250 ms budget is set at 100k rows (concept §18)",
|
||||
"review_by": "2027-02-17",
|
||||
},
|
||||
("huge", "library_stats", "latency_max_ms"): {
|
||||
"limit": 4_000,
|
||||
"approved_by": "domverse",
|
||||
"reason": "measured 3.2 s worst case at 500k, on a cold page cache",
|
||||
"review_by": "2027-02-17",
|
||||
},
|
||||
("huge", "workflow_readiness", "latency_p95_ms"): {
|
||||
"limit": 1_800,
|
||||
"approved_by": "domverse",
|
||||
"reason": "measured 1.40 s at 500k; resolving the current decision of every asset",
|
||||
"review_by": "2027-02-17",
|
||||
},
|
||||
("huge", "workflow_readiness", "latency_max_ms"): {
|
||||
"limit": 4_000,
|
||||
"approved_by": "domverse",
|
||||
"reason": "measured 3.3 s worst case at 500k, on a cold page cache",
|
||||
"review_by": "2027-02-17",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
# ── resource sampling ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def rss_bytes() -> int:
|
||||
"""Resident set size of this process, without a psutil dependency."""
|
||||
usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
||||
# Linux reports kilobytes, BSD/macOS bytes.
|
||||
return usage if usage > 1 << 32 or os.uname().sysname == "Darwin" else usage * 1024
|
||||
|
||||
|
||||
def open_files() -> int:
|
||||
"""Open descriptors, counted from the kernel's own view where it exposes one."""
|
||||
for directory in ("/proc/self/fd", "/dev/fd"):
|
||||
try:
|
||||
return len(os.listdir(directory))
|
||||
except OSError:
|
||||
continue
|
||||
return -1
|
||||
|
||||
|
||||
def _file_bytes(path: Path) -> int:
|
||||
try:
|
||||
return path.stat().st_size
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
|
||||
def _tree_bytes(path: Path) -> int:
|
||||
if not path.is_dir():
|
||||
return 0
|
||||
return sum(p.stat().st_size for p in path.rglob("*") if p.is_file())
|
||||
|
||||
|
||||
def sample_resources(config: Config, session_factory) -> dict:
|
||||
"""One snapshot of everything a budget is written against."""
|
||||
database = config.database_path
|
||||
with session_factory() as session:
|
||||
queue_depth = int(
|
||||
session.scalar(select(func.count()).select_from(Job).where(Job.state.in_(ACTIVE_STATES)))
|
||||
or 0
|
||||
)
|
||||
events = int(session.scalar(select(func.count()).select_from(JobEvent)) or 0)
|
||||
cache_bytes = _tree_bytes(config.thumbnail_cache_dir)
|
||||
return {
|
||||
"at": _now().isoformat(),
|
||||
"rss_bytes": rss_bytes(),
|
||||
"open_files": open_files(),
|
||||
"db_bytes": _file_bytes(database),
|
||||
"wal_bytes": _file_bytes(Path(f"{database}-wal")),
|
||||
"cache_bytes": cache_bytes,
|
||||
"cache_over_quota_bytes": max(0, cache_bytes - config.thumbnail_cache_quota_bytes),
|
||||
"queue_depth": queue_depth,
|
||||
"event_rows": events,
|
||||
}
|
||||
|
||||
|
||||
# ── synthetic library ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def synthesize(config: Config, *, assets: int, cluster_members: int, batch: int = 5_000) -> dict:
|
||||
"""Build a database of ``assets`` rows and one cluster of ``cluster_members``.
|
||||
|
||||
Rows only — no image files. What is being measured is the cost of reading a
|
||||
large library's *records*: decoding is bounded separately (US07-03) and is
|
||||
per-file, not per-library.
|
||||
"""
|
||||
config.database_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
run_migrations(config.database_url)
|
||||
engine = create_db_engine(config.database_url)
|
||||
factory = create_session_factory(engine)
|
||||
started = time.monotonic()
|
||||
root = config.library_roots[0] if config.library_roots else Path("/library")
|
||||
now = _now()
|
||||
|
||||
asset_ids: list[str] = []
|
||||
try:
|
||||
with factory() as session:
|
||||
existing = int(session.scalar(select(func.count()).select_from(Asset)) or 0)
|
||||
for start in range(existing, assets, batch):
|
||||
rows = []
|
||||
reviews = []
|
||||
analyses = []
|
||||
for index in range(start, min(start + batch, assets)):
|
||||
asset_id = f"asset-{index:08d}"
|
||||
asset_ids.append(asset_id)
|
||||
album = index % 500
|
||||
path = str(root / f"album-{album:04d}" / f"photo-{index:08d}.jpg")
|
||||
rows.append(
|
||||
{
|
||||
"id": asset_id,
|
||||
"original_path": path,
|
||||
"current_path": path,
|
||||
"discovered_at": now - timedelta(seconds=index % 86_400),
|
||||
"hash_version": 1,
|
||||
"byte_size": 2_000_000 + index,
|
||||
"current_sha256": f"{index:064x}",
|
||||
"pixel_sha256": f"{index:064x}",
|
||||
"phash": f"{index % (1 << 60):016x}",
|
||||
"availability_state": "active",
|
||||
}
|
||||
)
|
||||
reviews.append(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"asset_id": asset_id,
|
||||
"decision": "sfw" if index % 10 else "nsfw",
|
||||
"created_at": now,
|
||||
}
|
||||
)
|
||||
if index % 2 == 0: # half the library analysed, as in a real run
|
||||
analyses.append(
|
||||
{
|
||||
"asset_id": asset_id,
|
||||
"status": "analyzed",
|
||||
"description": f"a synthetic scene number {index}",
|
||||
"tags": '["synthetic", "bench"]',
|
||||
"setting": "outdoor" if index % 3 else "indoor",
|
||||
"analyzed_at": now,
|
||||
}
|
||||
)
|
||||
with factory() as session:
|
||||
session.execute(insert(Asset), rows)
|
||||
session.execute(insert(SafetyReview), reviews)
|
||||
if analyses:
|
||||
session.execute(insert(AnalysisResult), analyses)
|
||||
session.commit()
|
||||
|
||||
if cluster_members:
|
||||
with factory() as session:
|
||||
cluster_id = str(uuid.uuid4())
|
||||
session.add(
|
||||
DuplicateCluster(
|
||||
id=cluster_id,
|
||||
method="perceptual",
|
||||
confidence="near",
|
||||
state="open",
|
||||
version=1,
|
||||
)
|
||||
)
|
||||
session.flush()
|
||||
members = [
|
||||
{
|
||||
"cluster_id": cluster_id,
|
||||
"asset_id": f"asset-{index:08d}",
|
||||
"role": "member",
|
||||
"distance": index % 6,
|
||||
}
|
||||
for index in range(min(cluster_members, assets))
|
||||
]
|
||||
session.execute(insert(DuplicateMember), members)
|
||||
session.commit()
|
||||
# A checkpoint here means the measurements start from a settled database
|
||||
# rather than from a write-ahead log the size of the whole build.
|
||||
with sqlite3.connect(config.database_path) as connection:
|
||||
connection.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
finally:
|
||||
engine.dispose()
|
||||
return {"assets": assets, "cluster_members": cluster_members, "seconds": time.monotonic() - started}
|
||||
|
||||
|
||||
# ── scenarios ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class Scenario:
|
||||
name: str
|
||||
call: object
|
||||
iterations: int
|
||||
samples: list[float] = field(default_factory=list)
|
||||
|
||||
def run(self) -> dict:
|
||||
for _ in range(self.iterations):
|
||||
started = time.perf_counter()
|
||||
self.call()
|
||||
self.samples.append((time.perf_counter() - started) * 1000)
|
||||
ordered = sorted(self.samples)
|
||||
index = max(0, int(round(0.95 * len(ordered))) - 1)
|
||||
return {
|
||||
"scenario": self.name,
|
||||
"iterations": self.iterations,
|
||||
"latency_p50_ms": round(statistics.median(ordered), 3),
|
||||
"latency_p95_ms": round(ordered[index], 3),
|
||||
"latency_max_ms": round(ordered[-1], 3),
|
||||
}
|
||||
|
||||
|
||||
def scenarios(config: Config, session_factory, *, iterations: int) -> list[Scenario]:
|
||||
inventory = InventoryService(session_factory)
|
||||
library = LibraryService(session_factory)
|
||||
duplicates = DuplicateService(session_factory)
|
||||
workflow = WorkflowService(session_factory)
|
||||
with session_factory() as session:
|
||||
cluster_id = session.scalar(select(DuplicateCluster.id))
|
||||
|
||||
built = [
|
||||
Scenario("inventory_page", lambda: inventory.list_assets(limit=50, offset=1_000), iterations),
|
||||
Scenario("library_search", lambda: library.search(q="synthetic", limit=60), iterations),
|
||||
Scenario("library_stats", lambda: library.stats(), iterations),
|
||||
Scenario("workflow_readiness", lambda: workflow.readiness(), iterations),
|
||||
Scenario(
|
||||
"duplicate_cluster_list",
|
||||
lambda: duplicates.list_clusters(limit=50, offset=0),
|
||||
iterations,
|
||||
),
|
||||
]
|
||||
if cluster_id:
|
||||
built.append(
|
||||
Scenario(
|
||||
"duplicate_cluster_page",
|
||||
lambda: duplicates.get_cluster(cluster_id, limit=100, offset=0),
|
||||
iterations,
|
||||
)
|
||||
)
|
||||
return built
|
||||
|
||||
|
||||
# ── budget evaluation ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def evaluate(profile: str, measurements: list[dict]) -> tuple[list[dict], list[dict]]:
|
||||
"""Compare measurements with the budgets. Returns ``(breaches, exceptions_used)``."""
|
||||
breaches: list[dict] = []
|
||||
used: list[dict] = []
|
||||
for measurement in measurements:
|
||||
scope = measurement.get("scenario", "resources")
|
||||
for budget in BUDGETS:
|
||||
if budget.metric not in measurement:
|
||||
continue
|
||||
value = measurement[budget.metric]
|
||||
if value is None or value < 0:
|
||||
continue
|
||||
limit = budget.limit
|
||||
exception = APPROVED_EXCEPTIONS.get((profile, scope, budget.metric))
|
||||
if exception:
|
||||
limit = exception["limit"]
|
||||
used.append({"scope": scope, "metric": budget.metric, **exception})
|
||||
if value > limit:
|
||||
breaches.append(
|
||||
{
|
||||
"scope": scope,
|
||||
"metric": budget.metric,
|
||||
"value": value,
|
||||
"limit": limit,
|
||||
"unit": budget.unit,
|
||||
"why": budget.why,
|
||||
}
|
||||
)
|
||||
return breaches, used
|
||||
|
||||
|
||||
# ── soak ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def soak(config: Config, session_factory, *, seconds: float, interval: float = 1.0) -> dict:
|
||||
"""Browse, queue, cancel, and retry for a while; watch what grows.
|
||||
|
||||
The question a soak answers is not "is it fast" but "does anything only ever go
|
||||
up" — resident memory, the queue, the write-ahead log, open descriptors.
|
||||
"""
|
||||
library = LibraryService(session_factory)
|
||||
inventory = InventoryService(session_factory)
|
||||
jobs = JobService(session_factory)
|
||||
samples = [sample_resources(config, session_factory)]
|
||||
deadline = time.monotonic() + seconds
|
||||
last_sample = time.monotonic()
|
||||
cycles = 0
|
||||
while time.monotonic() < deadline:
|
||||
offset = (cycles * 50) % 1_000
|
||||
library.search(q="synthetic", limit=60, offset=offset)
|
||||
inventory.list_assets(limit=50, offset=offset)
|
||||
job = jobs.enqueue("scan", items=[f"soak-{cycles}"])
|
||||
jobs.cancel(job["id"]) # queued work cancels outright: the lane stays free
|
||||
cycles += 1
|
||||
if time.monotonic() - last_sample >= interval:
|
||||
gc.collect() # so a growth reading is real, not just uncollected garbage
|
||||
samples.append(sample_resources(config, session_factory))
|
||||
last_sample = time.monotonic()
|
||||
samples.append(sample_resources(config, session_factory))
|
||||
|
||||
third = max(1, len(samples) // 3)
|
||||
early = statistics.mean(sample["rss_bytes"] for sample in samples[:third])
|
||||
late = statistics.mean(sample["rss_bytes"] for sample in samples[-third:])
|
||||
return {
|
||||
"scenario": "soak",
|
||||
"seconds": seconds,
|
||||
"cycles": cycles,
|
||||
"samples": samples,
|
||||
"rss_growth_bytes": max(0, int(late - early)),
|
||||
"queue_depth": max(sample["queue_depth"] for sample in samples),
|
||||
"wal_bytes": max(sample["wal_bytes"] for sample in samples),
|
||||
"open_files": max(sample["open_files"] for sample in samples),
|
||||
"cache_over_quota_bytes": max(sample["cache_over_quota_bytes"] for sample in samples),
|
||||
}
|
||||
|
||||
|
||||
# ── the run ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def run(
|
||||
config: Config,
|
||||
*,
|
||||
profile: str = "smoke",
|
||||
soak_seconds: float = 0.0,
|
||||
output: Path | str | None = None,
|
||||
) -> dict:
|
||||
"""Build, measure, evaluate. Returns the report; the caller decides the exit code."""
|
||||
if profile not in PROFILES:
|
||||
raise ValueError(f"unknown profile {profile!r}; try one of {sorted(PROFILES)}")
|
||||
settings = PROFILES[profile]
|
||||
report = {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"profile": profile,
|
||||
"started_at": _now().isoformat(),
|
||||
"budgets": [
|
||||
{"metric": b.metric, "limit": b.limit, "unit": b.unit, "why": b.why} for b in BUDGETS
|
||||
],
|
||||
"runs": [],
|
||||
}
|
||||
measurements: list[dict] = []
|
||||
|
||||
for size in settings["sizes"]:
|
||||
sized = config.model_copy(update={"data_dir": Path(config.data_dir) / f"bench-{size}"})
|
||||
before = None
|
||||
build = synthesize(
|
||||
sized, assets=size, cluster_members=settings["cluster_members"]
|
||||
)
|
||||
engine = create_db_engine(sized.database_url)
|
||||
factory = create_session_factory(engine)
|
||||
try:
|
||||
before = sample_resources(sized, factory)
|
||||
results = [
|
||||
scenario.run()
|
||||
for scenario in scenarios(sized, factory, iterations=settings["iterations"])
|
||||
]
|
||||
after = sample_resources(sized, factory)
|
||||
after["scenario"] = "resources"
|
||||
after["rss_growth_bytes"] = max(0, after["rss_bytes"] - before["rss_bytes"])
|
||||
soaked = (
|
||||
soak(sized, factory, seconds=soak_seconds) if soak_seconds > 0 else None
|
||||
)
|
||||
finally:
|
||||
engine.dispose()
|
||||
measurements.extend(results)
|
||||
measurements.append(after)
|
||||
if soaked:
|
||||
measurements.append(soaked)
|
||||
report["runs"].append(
|
||||
{
|
||||
"assets": size,
|
||||
"build": build,
|
||||
"before": before,
|
||||
"scenarios": results,
|
||||
"resources": after,
|
||||
"soak": soaked,
|
||||
}
|
||||
)
|
||||
|
||||
breaches, exceptions_used = evaluate(profile, measurements)
|
||||
report["breaches"] = breaches
|
||||
report["exceptions_applied"] = exceptions_used
|
||||
report["ok"] = not breaches
|
||||
report["finished_at"] = _now().isoformat()
|
||||
if output:
|
||||
path = Path(output)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(report, indent=2))
|
||||
return report
|
||||
@@ -52,6 +52,13 @@ from photo_pipeline.services import availability, hashing
|
||||
NEAR_MAX = 5
|
||||
SIMILAR_MAX = 10
|
||||
|
||||
# Member paging (US07-06). A burst or a re-imported folder can put thousands of
|
||||
# assets in one cluster; review looks at a few at a time, so neither the list view
|
||||
# nor the detail view may load them all.
|
||||
MEMBER_PAGE = 100
|
||||
MAX_MEMBER_PAGE = 500
|
||||
SNAPSHOT_MEMBER_PREVIEW = 20
|
||||
|
||||
|
||||
class Method(str, Enum):
|
||||
EXACT = "exact"
|
||||
@@ -520,39 +527,65 @@ class DuplicateService:
|
||||
items = [self._snapshot(session, c.id) for c in rows]
|
||||
return {"items": items, "total": int(total or 0), "limit": limit, "offset": offset}
|
||||
|
||||
def get_cluster(self, cluster_id: str) -> dict | None:
|
||||
"""Cluster detail enriched with per-member asset evidence for comparison."""
|
||||
def get_cluster(
|
||||
self, cluster_id: str, *, limit: int = MEMBER_PAGE, offset: int = 0
|
||||
) -> dict | None:
|
||||
"""Cluster detail enriched with per-member asset evidence for comparison.
|
||||
|
||||
Members are paged and their evidence is loaded in batches (US07-06). A
|
||||
cluster of a few thousand near-identical frames is a real shape for a phone
|
||||
library, and the review screen only ever shows a handful at a time: loading
|
||||
every member — each with its own asset, thumbnail, and location query — made
|
||||
opening such a cluster cost thousands of round trips and megabytes of JSON.
|
||||
"""
|
||||
limit = max(1, min(limit, MAX_MEMBER_PAGE))
|
||||
offset = max(0, offset)
|
||||
with self._session_factory() as session:
|
||||
cluster = session.get(DuplicateCluster, cluster_id)
|
||||
if cluster is None:
|
||||
return None
|
||||
member_total = int(
|
||||
session.scalar(
|
||||
select(func.count())
|
||||
.select_from(DuplicateMember)
|
||||
.where(DuplicateMember.cluster_id == cluster_id)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
rows = list(
|
||||
session.execute(
|
||||
select(DuplicateMember)
|
||||
.where(DuplicateMember.cluster_id == cluster_id)
|
||||
.order_by(DuplicateMember.asset_id)
|
||||
.limit(limit)
|
||||
.offset(offset)
|
||||
).scalars()
|
||||
)
|
||||
evidence = self._member_evidence(session, [row.asset_id for row in rows])
|
||||
members = []
|
||||
for member in session.execute(
|
||||
select(DuplicateMember).where(DuplicateMember.cluster_id == cluster_id)
|
||||
).scalars():
|
||||
asset = session.get(Asset, member.asset_id)
|
||||
for member in rows:
|
||||
try:
|
||||
evidence = json.loads(member.evidence) if member.evidence else {}
|
||||
member_evidence = json.loads(member.evidence) if member.evidence else {}
|
||||
except json.JSONDecodeError:
|
||||
evidence = {}
|
||||
member_evidence = {}
|
||||
asset, offline = evidence[member.asset_id]
|
||||
members.append(
|
||||
{
|
||||
"asset_id": member.asset_id,
|
||||
"role": member.role,
|
||||
"distance": member.distance,
|
||||
"evidence": evidence,
|
||||
"evidence": member_evidence,
|
||||
"current_path": asset.current_path if asset else None,
|
||||
"byte_size": asset.byte_size if asset else None,
|
||||
"phash": asset.phash if asset else None,
|
||||
**self._offline_evidence(session, asset),
|
||||
**offline,
|
||||
}
|
||||
)
|
||||
members.sort(key=lambda m: m["asset_id"])
|
||||
# A full-resolution comparison of an offline original is impossible; the
|
||||
# UI asks for that named medium instead of guessing (concept §9).
|
||||
mount_required = sorted(
|
||||
{m["archive_location"] for m in members if m["requires_mount"]}
|
||||
)
|
||||
# UI asks for that named medium instead of guessing (concept §9). The
|
||||
# answer covers the whole cluster, not just this page, so a mount is not
|
||||
# discovered halfway through a review.
|
||||
mount_required = self._mount_required(session, cluster_id)
|
||||
return {
|
||||
"id": cluster.id,
|
||||
"method": cluster.method,
|
||||
@@ -564,10 +597,74 @@ class DuplicateService:
|
||||
"requires_confirmation": cluster.method == Method.PERCEPTUAL.value,
|
||||
"mount_required": mount_required,
|
||||
"members": members,
|
||||
"member_total": member_total,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
}
|
||||
|
||||
def _offline_evidence(self, session, asset: Asset | None) -> dict:
|
||||
"""What review can still rely on when a member's original is not readable."""
|
||||
def _member_evidence(self, session, asset_ids: list[str]) -> dict:
|
||||
"""``{asset_id: (asset, offline_evidence)}`` for one page, in three queries."""
|
||||
if not asset_ids:
|
||||
return {}
|
||||
assets = {
|
||||
asset.id: asset
|
||||
for asset in session.execute(
|
||||
select(Asset).where(Asset.id.in_(asset_ids))
|
||||
).scalars()
|
||||
}
|
||||
previews: dict[str, list] = {}
|
||||
for thumbnail in session.execute(
|
||||
select(Thumbnail).where(Thumbnail.asset_id.in_(asset_ids))
|
||||
).scalars():
|
||||
previews.setdefault(thumbnail.asset_id, []).append(thumbnail)
|
||||
location_ids = {
|
||||
asset.archive_location_id for asset in assets.values() if asset.archive_location_id
|
||||
}
|
||||
locations = (
|
||||
{
|
||||
location.id: location
|
||||
for location in session.execute(
|
||||
select(ArchiveLocation).where(ArchiveLocation.id.in_(location_ids))
|
||||
).scalars()
|
||||
}
|
||||
if location_ids
|
||||
else {}
|
||||
)
|
||||
return {
|
||||
asset_id: (
|
||||
assets.get(asset_id),
|
||||
self._offline_evidence(
|
||||
assets.get(asset_id),
|
||||
locations=locations,
|
||||
thumbnails=previews.get(asset_id, []),
|
||||
),
|
||||
)
|
||||
for asset_id in asset_ids
|
||||
}
|
||||
|
||||
def _mount_required(self, session, cluster_id: str) -> list[str]:
|
||||
"""Archive media whose originals this cluster needs, across every member."""
|
||||
rows = session.execute(
|
||||
select(ArchiveLocation.name)
|
||||
.select_from(DuplicateMember)
|
||||
.join(Asset, Asset.id == DuplicateMember.asset_id)
|
||||
.join(ArchiveLocation, ArchiveLocation.id == Asset.archive_location_id)
|
||||
.where(
|
||||
DuplicateMember.cluster_id == cluster_id,
|
||||
Asset.availability_state == availability.ARCHIVED_OFFLINE,
|
||||
)
|
||||
.distinct()
|
||||
).scalars()
|
||||
return sorted(rows)
|
||||
|
||||
def _offline_evidence(
|
||||
self, asset: Asset | None, *, locations: dict, thumbnails: list
|
||||
) -> dict:
|
||||
"""What review can still rely on when a member's original is not readable.
|
||||
|
||||
Takes the already-loaded locations and thumbnails for its page rather than
|
||||
querying per member (US07-06).
|
||||
"""
|
||||
if asset is None:
|
||||
return {
|
||||
"availability_state": None,
|
||||
@@ -577,12 +674,8 @@ class DuplicateService:
|
||||
"preview": {"state": "missing", "protected": False},
|
||||
"requires_mount": False,
|
||||
}
|
||||
location = (
|
||||
session.get(ArchiveLocation, asset.archive_location_id)
|
||||
if asset.archive_location_id
|
||||
else None
|
||||
)
|
||||
preview = self._preview_evidence(session, asset.id)
|
||||
location = locations.get(asset.archive_location_id)
|
||||
preview = self._preview_evidence(thumbnails)
|
||||
archived = asset.availability_state in availability.ARCHIVED
|
||||
return {
|
||||
"availability_state": asset.availability_state,
|
||||
@@ -598,10 +691,7 @@ class DuplicateService:
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _preview_evidence(session, asset_id: str) -> dict:
|
||||
rows = list(
|
||||
session.execute(select(Thumbnail).where(Thumbnail.asset_id == asset_id)).scalars()
|
||||
)
|
||||
def _preview_evidence(rows: list) -> dict:
|
||||
ready = [r for r in rows if r.state == "ready" and r.path]
|
||||
if ready:
|
||||
best = max(ready, key=lambda r: (bool(r.protected), r.size or 0))
|
||||
@@ -693,11 +783,29 @@ class DuplicateService:
|
||||
session.delete(link)
|
||||
|
||||
def _snapshot(self, session, cluster_id) -> dict:
|
||||
"""A cluster and a *bounded* preview of its members.
|
||||
|
||||
The list view shows a count and a few ids; a snapshot that loaded every
|
||||
member turned one page of 200 clusters into hundreds of thousands of rows
|
||||
(US07-06). ``member_total`` is the honest count either way.
|
||||
"""
|
||||
cluster = session.get(DuplicateCluster, cluster_id)
|
||||
member_total = int(
|
||||
session.scalar(
|
||||
select(func.count())
|
||||
.select_from(DuplicateMember)
|
||||
.where(DuplicateMember.cluster_id == cluster_id)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
members = session.execute(
|
||||
select(DuplicateMember).where(DuplicateMember.cluster_id == cluster_id)
|
||||
select(DuplicateMember)
|
||||
.where(DuplicateMember.cluster_id == cluster_id)
|
||||
.order_by(DuplicateMember.asset_id)
|
||||
.limit(SNAPSHOT_MEMBER_PREVIEW)
|
||||
).scalars()
|
||||
return {
|
||||
"member_total": member_total,
|
||||
"id": cluster.id,
|
||||
"method": cluster.method,
|
||||
"confidence": cluster.confidence,
|
||||
|
||||
@@ -15,10 +15,9 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy import String, and_, case, cast, func, or_, select, text
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from photo_pipeline.models import AnalysisResult, Asset
|
||||
@@ -71,38 +70,75 @@ class LibraryService:
|
||||
return {"rows": rows, "total": total, "offset": offset, "limit": limit}
|
||||
|
||||
def stats(self) -> dict:
|
||||
"""Library-wide totals, aggregated in SQL (US07-06).
|
||||
|
||||
This page used to load every analysis row — object, tags, and all — to count
|
||||
them in Python, which cost half a second at 100k assets and grew from there.
|
||||
Only the album breakdown still walks rows, and only their path and status:
|
||||
SQLite has no ``dirname``, and two short strings per asset is cheap.
|
||||
"""
|
||||
with self._session_factory() as session:
|
||||
status = dict(
|
||||
session.execute(
|
||||
select(AnalysisResult.status, func.count()).group_by(AnalysisResult.status)
|
||||
).all()
|
||||
)
|
||||
rows = list(
|
||||
session.execute(
|
||||
select(AnalysisResult, Asset.current_path).join(
|
||||
Asset, Asset.id == AnalysisResult.asset_id
|
||||
)
|
||||
)
|
||||
)
|
||||
albums: dict[str, dict] = {}
|
||||
tag_counts: Counter = Counter()
|
||||
year_counts: Counter = Counter()
|
||||
people: Counter = Counter()
|
||||
errors = []
|
||||
for result, path in rows:
|
||||
for path, row_status in session.execute(
|
||||
select(Asset.current_path, AnalysisResult.status).join(
|
||||
Asset, Asset.id == AnalysisResult.asset_id
|
||||
)
|
||||
):
|
||||
album = _album_of(path)
|
||||
bucket = albums.setdefault(album, {"album": album, "done": 0, "total": 0})
|
||||
bucket["total"] += 1
|
||||
if result.status in DONE:
|
||||
if row_status in DONE:
|
||||
bucket["done"] += 1
|
||||
for tag in _tags(result.tags):
|
||||
tag_counts[tag] += 1
|
||||
if result.approx_year is not None:
|
||||
year_counts[result.approx_year] += 1
|
||||
if result.people_count is not None:
|
||||
people["3+" if result.people_count >= 3 else str(result.people_count)] += 1
|
||||
if result.status == "error":
|
||||
errors.append({"path": path, "error": result.error_message})
|
||||
year_counts = dict(
|
||||
session.execute(
|
||||
select(AnalysisResult.approx_year, func.count())
|
||||
.where(AnalysisResult.approx_year.is_not(None))
|
||||
.group_by(AnalysisResult.approx_year)
|
||||
).all()
|
||||
)
|
||||
people = dict(
|
||||
session.execute(
|
||||
select(
|
||||
case(
|
||||
(AnalysisResult.people_count >= 3, "3+"),
|
||||
else_=cast(AnalysisResult.people_count, String),
|
||||
),
|
||||
func.count(),
|
||||
)
|
||||
.where(AnalysisResult.people_count.is_not(None))
|
||||
.group_by(
|
||||
case(
|
||||
(AnalysisResult.people_count >= 3, "3+"),
|
||||
else_=cast(AnalysisResult.people_count, String),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
# SQLite's JSON1 counts the tag arrays where they are: parsing 50k JSON
|
||||
# strings in Python to keep the top 40 is the definition of doing work
|
||||
# the database already does. Malformed tags are skipped, not fatal.
|
||||
tag_counts = session.execute(
|
||||
text(
|
||||
"SELECT tag.value AS value, count(*) AS total "
|
||||
"FROM analysis_results, json_each(analysis_results.tags) AS tag "
|
||||
"WHERE analysis_results.tags IS NOT NULL "
|
||||
"AND json_valid(analysis_results.tags) "
|
||||
"GROUP BY tag.value ORDER BY total DESC, value LIMIT 40"
|
||||
)
|
||||
).all()
|
||||
errors = [
|
||||
{"path": path, "error": message}
|
||||
for path, message in session.execute(
|
||||
select(Asset.current_path, AnalysisResult.error_message)
|
||||
.join(Asset, Asset.id == AnalysisResult.asset_id)
|
||||
.where(AnalysisResult.status == "error")
|
||||
)
|
||||
]
|
||||
return {
|
||||
"total": sum(status.values()),
|
||||
"status": status,
|
||||
@@ -111,7 +147,7 @@ class LibraryService:
|
||||
"season": self._facet("season"),
|
||||
"people": [{"value": v, "count": n} for v, n in sorted(people.items())],
|
||||
"years": [{"value": y, "count": year_counts[y]} for y in sorted(year_counts)],
|
||||
"top_tags": [{"value": t, "count": n} for t, n in tag_counts.most_common(40)],
|
||||
"top_tags": [{"value": t, "count": n} for t, n in tag_counts],
|
||||
"albums": sorted(albums.values(), key=lambda d: d["album"]),
|
||||
"errors": sorted(errors, key=lambda e: e["path"] or ""),
|
||||
}
|
||||
|
||||
@@ -105,7 +105,7 @@ def exif_projection(decision: str) -> dict[str, list[str]]:
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import column, func, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from photo_pipeline.models import Asset, ExifProjection, SafetyReview
|
||||
@@ -139,8 +139,8 @@ class SafetyService:
|
||||
|
||||
# -- reads ----------------------------------------------------------------
|
||||
def _latest_by_asset(self, session) -> dict[str, SafetyReview]:
|
||||
# Latest row per asset. Small local scale: order ascending, let later rows
|
||||
# overwrite. ponytail: a windowed query if safety_reviews ever grows huge.
|
||||
"""The current review per asset, as ORM rows. Only for small, known sets —
|
||||
every library-wide caller uses ``latest_reviews()`` in SQL instead."""
|
||||
latest: dict[str, SafetyReview] = {}
|
||||
for review in session.scalars(select(SafetyReview).order_by(SafetyReview.created_at)):
|
||||
latest[review.asset_id] = review
|
||||
@@ -148,58 +148,99 @@ class SafetyService:
|
||||
|
||||
def current_decision(self, asset_id: str) -> str | None:
|
||||
with self._session_factory() as session:
|
||||
review = self._latest_by_asset(session).get(asset_id)
|
||||
return review.decision if review else None
|
||||
latest = latest_reviews().subquery()
|
||||
return session.scalar(
|
||||
select(latest.c.decision).where(latest.c.asset_id == asset_id)
|
||||
)
|
||||
|
||||
def counts(self) -> dict[str, int]:
|
||||
"""Decision breakdown over canonical, active assets — the workflow totals."""
|
||||
"""Decision breakdown over canonical, active assets — the workflow totals.
|
||||
|
||||
Aggregated in SQL: the workflow home asks for this on every load, and
|
||||
materialising every asset and every review to count them cost hundreds of
|
||||
milliseconds at 25k assets and would scale linearly from there (US07-06).
|
||||
"""
|
||||
latest = latest_reviews().subquery()
|
||||
with self._session_factory() as session:
|
||||
assets = list(session.scalars(_eligible_assets_query()))
|
||||
latest = self._latest_by_asset(session)
|
||||
out = {SFW: 0, NSFW: 0, "deferred": 0, "undecided": 0, "scored": 0}
|
||||
for asset in assets:
|
||||
review = latest.get(asset.id)
|
||||
decision = review.decision if review else None
|
||||
if decision in (SFW, NSFW, "deferred"):
|
||||
out[decision] += 1
|
||||
else:
|
||||
out["undecided"] += 1
|
||||
if review and review.score is not None:
|
||||
out["scored"] += 1
|
||||
return out
|
||||
rows = session.execute(
|
||||
select(
|
||||
func.coalesce(latest.c.decision, "undecided"),
|
||||
func.count(),
|
||||
func.count(latest.c.score),
|
||||
)
|
||||
.select_from(Asset)
|
||||
.join(latest, latest.c.asset_id == Asset.id, isouter=True)
|
||||
.where(
|
||||
Asset.canonical_asset_id.is_(None),
|
||||
Asset.availability_state == "active",
|
||||
)
|
||||
.group_by(func.coalesce(latest.c.decision, "undecided"))
|
||||
).all()
|
||||
out = {SFW: 0, NSFW: 0, "deferred": 0, "undecided": 0, "scored": 0}
|
||||
for decision, total, scored in rows:
|
||||
if decision in (SFW, NSFW, "deferred"):
|
||||
out[decision] += int(total)
|
||||
else:
|
||||
# Anything that is not one of the three decisions is undecided —
|
||||
# including a score-only review, which is what "scored" counts.
|
||||
out["undecided"] += int(total)
|
||||
out["scored"] += int(scored)
|
||||
return out
|
||||
|
||||
def review_queue(self, state: str = "", limit: int = 100, offset: int = 0) -> dict:
|
||||
"""Assets for the review UI, filtered by ``state`` (undecided/sfw/nsfw/deferred)."""
|
||||
"""Assets for the review UI, filtered by ``state`` (undecided/sfw/nsfw/deferred).
|
||||
|
||||
Filtered, counted, and paged in SQL (US07-06): the queue for a large library
|
||||
is thousands of rows and the reviewer sees one page of it.
|
||||
"""
|
||||
latest = latest_reviews().subquery()
|
||||
projections = (
|
||||
select(ExifProjection.asset_id, ExifProjection.state.label("exif_state"))
|
||||
.where(ExifProjection.stage == "safety")
|
||||
.subquery()
|
||||
)
|
||||
effective = func.coalesce(latest.c.decision, "undecided")
|
||||
query = (
|
||||
select(
|
||||
Asset.id,
|
||||
Asset.current_path,
|
||||
latest.c.score,
|
||||
latest.c.decision,
|
||||
latest.c.exif_verified_at,
|
||||
projections.c.exif_state,
|
||||
)
|
||||
.select_from(Asset)
|
||||
.join(latest, latest.c.asset_id == Asset.id, isouter=True)
|
||||
.join(projections, projections.c.asset_id == Asset.id, isouter=True)
|
||||
.where(
|
||||
Asset.canonical_asset_id.is_(None),
|
||||
Asset.availability_state == "active",
|
||||
)
|
||||
)
|
||||
if state:
|
||||
query = query.where(effective == state)
|
||||
with self._session_factory() as session:
|
||||
assets = list(session.scalars(_eligible_assets_query().order_by(Asset.current_path)))
|
||||
latest = self._latest_by_asset(session)
|
||||
# One query, not one per asset: the reviewer needs to see a divergent
|
||||
# checkpoint, which is neither "verified" nor a plain failure (US07-03).
|
||||
projections = {
|
||||
row.asset_id: row.state
|
||||
for row in session.scalars(
|
||||
select(ExifProjection).where(ExifProjection.stage == "safety")
|
||||
)
|
||||
}
|
||||
rows = []
|
||||
for asset in assets:
|
||||
review = latest.get(asset.id)
|
||||
decision = review.decision if review else None
|
||||
effective = decision or "undecided"
|
||||
if state and state != effective:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"asset_id": asset.id,
|
||||
"current_path": asset.current_path,
|
||||
"score": review.score if review else None,
|
||||
"decision": decision,
|
||||
"suggested": classify(review.score) if review and review.score is not None else None,
|
||||
"exif_verified": bool(review and review.exif_verified_at),
|
||||
"exif_state": projections.get(asset.id),
|
||||
}
|
||||
)
|
||||
return {"total": len(rows), "items": rows[offset : offset + limit]}
|
||||
total = int(
|
||||
session.scalar(select(func.count()).select_from(query.subquery())) or 0
|
||||
)
|
||||
rows = session.execute(
|
||||
query.order_by(Asset.current_path).limit(limit).offset(offset)
|
||||
).all()
|
||||
return {
|
||||
"total": total,
|
||||
"items": [
|
||||
{
|
||||
"asset_id": asset_id,
|
||||
"current_path": current_path,
|
||||
"score": score,
|
||||
"decision": decision,
|
||||
"suggested": classify(score) if score is not None else None,
|
||||
"exif_verified": bool(exif_verified_at),
|
||||
"exif_state": exif_state,
|
||||
}
|
||||
for asset_id, current_path, score, decision, exif_verified_at, exif_state in rows
|
||||
],
|
||||
}
|
||||
|
||||
def scorable_asset_ids(self) -> list[str]:
|
||||
"""Canonical active assets with a path — the items a scoring job enqueues."""
|
||||
@@ -300,6 +341,34 @@ class SafetyService:
|
||||
}
|
||||
|
||||
|
||||
def latest_reviews():
|
||||
"""One row per asset: its current safety review, chosen in SQL.
|
||||
|
||||
``safety_reviews`` is append-only, so "the decision" is the newest row for an
|
||||
asset. A window function picks it without loading the table; ``rowid`` breaks a
|
||||
same-timestamp tie the same way the previous last-write-wins loop did.
|
||||
"""
|
||||
ranked = (
|
||||
select(
|
||||
SafetyReview.asset_id,
|
||||
SafetyReview.decision,
|
||||
SafetyReview.score,
|
||||
SafetyReview.exif_verified_at,
|
||||
func.row_number()
|
||||
.over(
|
||||
partition_by=SafetyReview.asset_id,
|
||||
order_by=(SafetyReview.created_at.desc(), column("rowid").desc()),
|
||||
)
|
||||
.label("rank"),
|
||||
)
|
||||
.select_from(SafetyReview)
|
||||
.subquery()
|
||||
)
|
||||
return select(
|
||||
ranked.c.asset_id, ranked.c.decision, ranked.c.score, ranked.c.exif_verified_at
|
||||
).where(ranked.c.rank == 1)
|
||||
|
||||
|
||||
def _eligible_assets_query():
|
||||
"""Canonical, active assets — the safety stage runs only on these.
|
||||
|
||||
|
||||
@@ -47,7 +47,9 @@ class WorkflowService:
|
||||
active = self._active_job(session)
|
||||
|
||||
safety = SafetyService(self._session_factory).counts()
|
||||
analysis = AnalysisService(self._session_factory).counts()
|
||||
# Reuse the confirmed-SFW total just computed: resolving the current decision
|
||||
# of every asset is the expensive part of this page (US07-06).
|
||||
analysis = AnalysisService(self._session_factory).counts(eligible=safety[_SFW])
|
||||
undecided_clusters = cluster_states.get("open", 0) + cluster_states.get("reopened", 0)
|
||||
|
||||
stages = [
|
||||
|
||||
Reference in New Issue
Block a user