diff --git a/.gitignore b/.gitignore index ed83884..3f58b27 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ htmlcov/ *.db-* *.sqlite* *.log +*_history.jsonl data/ downloads/ _archive/ diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..dd64d11 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,5 @@ +# Alembic configuration. The database URL and script location are supplied at +# runtime by photo_pipeline.db.run_migrations, so they are intentionally left +# unset here. Logging is configured by the application, not by Alembic. +[alembic] +script_location = migrations diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..075c403 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,50 @@ +"""Alembic environment. + +The database URL is injected by photo_pipeline.db.run_migrations. Importing the +models package registers every table on Base.metadata for autogenerate. Logging +config is owned by the application, so fileConfig is intentionally not called. +""" + +from __future__ import annotations + +from alembic import context +from sqlalchemy import engine_from_config, pool + +import photo_pipeline.models # noqa: F401 (registers models on Base.metadata) +from photo_pipeline.db import Base + +config = context.config +target_metadata = Base.metadata + + +def run_migrations_offline() -> None: + context.configure( + url=config.get_main_option("sqlalchemy.url"), + target_metadata=target_metadata, + literal_binds=True, + render_as_batch=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + render_as_batch=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..36cd8ef --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,22 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/0001_initial_identity.py b/migrations/versions/0001_initial_identity.py new file mode 100644 index 0000000..fc5bd02 --- /dev/null +++ b/migrations/versions/0001_initial_identity.py @@ -0,0 +1,70 @@ +"""Initial stable-identity schema: assets and asset_paths. + +Revision ID: 0001_initial_identity +Revises: +Create Date: 2026-07-15 +""" + +import sqlalchemy as sa +from alembic import op + +revision = "0001_initial_identity" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "assets", + sa.Column("id", sa.String(), primary_key=True), + sa.Column("original_path", sa.String(), nullable=False), + sa.Column("current_path", sa.String(), nullable=True, unique=True), + sa.Column("current_sha256", sa.String(), nullable=True), + sa.Column("pixel_sha256", sa.String(), nullable=True), + sa.Column("phash", sa.String(), nullable=True), + sa.Column("hash_version", sa.Integer(), nullable=False, server_default="1"), + sa.Column("byte_size", sa.Integer(), nullable=True), + sa.Column("discovered_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("missing_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "availability_state", sa.String(), nullable=False, server_default="active" + ), + sa.Column("state_version", sa.Integer(), nullable=False, server_default="1"), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + ) + op.create_index("ix_assets_current_sha256", "assets", ["current_sha256"]) + op.create_index("ix_assets_pixel_sha256", "assets", ["pixel_sha256"]) + op.create_index("ix_assets_phash", "assets", ["phash"]) + op.create_index("ix_assets_availability_state", "assets", ["availability_state"]) + + op.create_table( + "asset_paths", + sa.Column( + "asset_id", + sa.String(), + sa.ForeignKey("assets.id"), + primary_key=True, + ), + sa.Column("path", sa.String(), primary_key=True), + sa.Column("valid_from", sa.DateTime(timezone=True), primary_key=True), + sa.Column("valid_until", sa.DateTime(timezone=True), nullable=True), + sa.Column("reason", sa.String(), nullable=True), + ) + op.create_index("ix_asset_paths_path", "asset_paths", ["path"]) + + +def downgrade() -> None: + op.drop_table("asset_paths") + op.drop_table("assets") diff --git a/photo_pipeline/__init__.py b/photo_pipeline/__init__.py new file mode 100644 index 0000000..1a55c05 --- /dev/null +++ b/photo_pipeline/__init__.py @@ -0,0 +1,9 @@ +"""Photo pipeline application package. + +Target architecture for the integrated photo-library pipeline (see +INTEGRATED_PIPELINE_CONCEPT.md). US01-02 establishes the foundation: typed +configuration, database engine/session lifecycle, versioned migrations, the +FastAPI application factory, structured logging, and readiness. +""" + +__version__ = "0.1.0" diff --git a/photo_pipeline/__main__.py b/photo_pipeline/__main__.py new file mode 100644 index 0000000..4486d6e --- /dev/null +++ b/photo_pipeline/__main__.py @@ -0,0 +1,35 @@ +"""Application management CLI: ``python -m photo_pipeline {serve,migrate}``.""" + +from __future__ import annotations + +import argparse +from typing import Sequence + +from photo_pipeline.config import Config +from photo_pipeline.db import run_migrations + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="photo_pipeline") + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("serve", help="Run the API server") + commands.add_parser("migrate", help="Upgrade the database to the latest revision") + args = parser.parse_args(argv) + + config = Config.from_env() + config.database_path.parent.mkdir(parents=True, exist_ok=True) + + if args.command == "migrate": + run_migrations(config.database_url) + return 0 + + import uvicorn + + from photo_pipeline.api.app import create_app + + uvicorn.run(create_app(config), host=config.host, port=config.port) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/photo_pipeline/api/__init__.py b/photo_pipeline/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/photo_pipeline/api/app.py b/photo_pipeline/api/app.py new file mode 100644 index 0000000..7ec03a2 --- /dev/null +++ b/photo_pipeline/api/app.py @@ -0,0 +1,41 @@ +"""FastAPI application factory and lifecycle. + +Startup runs migrations and opens the database engine; shutdown disposes it so no +connection or engine resource leaks. The engine, session factory, and config live +on ``app.state`` for dependencies to use. The app binds to 127.0.0.1 by default +and exposes the versioned ``/api/v1`` surface; US01-02 ships only health. +""" + +from __future__ import annotations + +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from photo_pipeline.api.routes import health +from photo_pipeline.config import Config +from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations +from photo_pipeline.logging import configure_logging + + +def create_app(config: Config | None = None) -> FastAPI: + config = config or Config.from_env() + configure_logging(config.log_level, config.log_format) + + @asynccontextmanager + async def lifespan(app: FastAPI): + config.database_path.parent.mkdir(parents=True, exist_ok=True) + run_migrations(config.database_url) + engine = create_db_engine(config.database_url) + app.state.config = config + app.state.engine = engine + app.state.session_factory = create_session_factory(engine) + try: + yield + finally: + engine.dispose() + app.state.engine = None + + app = FastAPI(title="Photo Pipeline", version="0.1.0", lifespan=lifespan) + app.include_router(health.router, prefix="/api/v1") + return app diff --git a/photo_pipeline/api/routes/__init__.py b/photo_pipeline/api/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/photo_pipeline/api/routes/health.py b/photo_pipeline/api/routes/health.py new file mode 100644 index 0000000..38569e6 --- /dev/null +++ b/photo_pipeline/api/routes/health.py @@ -0,0 +1,45 @@ +"""Liveness and readiness endpoints. + +``/health/live`` answers whether the process is up. ``/health/ready`` answers +whether it can serve: the database is reachable and configured correctly (WAL, +foreign keys). Tests and any future orchestration wait on readiness rather than a +sleep. Readiness returns 503 until the checks pass. +""" + +from __future__ import annotations + +from fastapi import APIRouter, Request, Response +from sqlalchemy import text + +router = APIRouter(tags=["health"]) + + +@router.get("/health/live") +def live() -> dict: + return {"status": "alive"} + + +@router.get("/health/ready") +def ready(request: Request, response: Response) -> dict: + checks: dict[str, object] = {} + ok = True + engine = getattr(request.app.state, "engine", None) + if engine is None: + ok = False + checks["database"] = "unavailable" + else: + try: + with engine.connect() as conn: + conn.execute(text("SELECT 1")) + journal_mode = conn.execute(text("PRAGMA journal_mode")).scalar() + foreign_keys = conn.execute(text("PRAGMA foreign_keys")).scalar() + checks["database"] = "ok" + checks["journal_mode"] = journal_mode + checks["foreign_keys"] = int(foreign_keys) + if str(journal_mode).lower() != "wal" or int(foreign_keys) != 1: + ok = False + except Exception: + ok = False + checks["database"] = "error" + response.status_code = 200 if ok else 503 + return {"status": "ready" if ok else "not_ready", "checks": checks} diff --git a/photo_pipeline/config.py b/photo_pipeline/config.py new file mode 100644 index 0000000..834545a --- /dev/null +++ b/photo_pipeline/config.py @@ -0,0 +1,52 @@ +"""Typed application configuration. + +Values come from ``PHOTO_PIPELINE_*`` environment variables. Secrets use +``SecretStr`` so they are masked in logs, reprs, and model dumps, and are never +returned by the API. Only ``.get_secret_value()`` exposes them, and only where a +real external call needs them. + +pydantic-settings would do this too, but a prefix-scan over the declared fields +is a few lines and one fewer dependency. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Mapping + +from pydantic import BaseModel, ConfigDict, SecretStr + +ENV_PREFIX = "PHOTO_PIPELINE_" + + +class Config(BaseModel): + model_config = ConfigDict(frozen=True) + + data_dir: Path = Path("data") + db_path: Path | None = None # defaults to data_dir/photo_pipeline.db + host: str = "127.0.0.1" + port: int = 8000 + log_level: str = "INFO" + log_format: str = "json" # "json" or "text" + + vision_api_key: SecretStr | None = None + immich_api_key: SecretStr | None = None + + @property + def database_path(self) -> Path: + return self.db_path or (self.data_dir / "photo_pipeline.db") + + @property + def database_url(self) -> str: + return f"sqlite:///{self.database_path}" + + @classmethod + def from_env(cls, environ: Mapping[str, str] | None = None) -> "Config": + env = os.environ if environ is None else environ + data = { + name: env[ENV_PREFIX + name.upper()] + for name in cls.model_fields + if env.get(ENV_PREFIX + name.upper()) + } + return cls(**data) diff --git a/photo_pipeline/db.py b/photo_pipeline/db.py new file mode 100644 index 0000000..b8a7eb6 --- /dev/null +++ b/photo_pipeline/db.py @@ -0,0 +1,52 @@ +"""Database engine, sessions, and migration entry point. + +SQLite is opened in WAL mode with foreign keys enforced on every connection +(foreign keys and busy_timeout are per-connection; WAL is persistent but cheap to +re-assert). Migrations run through Alembic so the same versioned schema is used at +startup and in tests. Repository and service code depends only on ``Base`` and the +session factory, never on SQLite specifics, so PostgreSQL stays a future option. +""" + +from __future__ import annotations + +from pathlib import Path + +from sqlalchemy import Engine, create_engine, event +from sqlalchemy.orm import DeclarativeBase, sessionmaker + +_REPO_ROOT = Path(__file__).resolve().parent.parent + + +class Base(DeclarativeBase): + pass + + +def create_db_engine(url: str, *, echo: bool = False) -> Engine: + connect_args = {"check_same_thread": False} if url.startswith("sqlite") else {} + engine = create_engine(url, echo=echo, future=True, connect_args=connect_args) + if url.startswith("sqlite"): + event.listen(engine, "connect", _apply_sqlite_pragmas) + return engine + + +def _apply_sqlite_pragmas(dbapi_connection, _connection_record) -> None: + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA journal_mode=WAL") + cursor.execute("PRAGMA foreign_keys=ON") + cursor.execute("PRAGMA busy_timeout=5000") + cursor.close() + + +def create_session_factory(engine: Engine) -> sessionmaker: + return sessionmaker(bind=engine, expire_on_commit=False, future=True) + + +def run_migrations(url: str) -> None: + """Upgrade the database at ``url`` to the latest revision.""" + from alembic import command + from alembic.config import Config as AlembicConfig + + cfg = AlembicConfig(str(_REPO_ROOT / "alembic.ini")) + cfg.set_main_option("script_location", str(_REPO_ROOT / "migrations")) + cfg.set_main_option("sqlalchemy.url", url) + command.upgrade(cfg, "head") diff --git a/photo_pipeline/logging.py b/photo_pipeline/logging.py new file mode 100644 index 0000000..9288a4e --- /dev/null +++ b/photo_pipeline/logging.py @@ -0,0 +1,44 @@ +"""Structured logging. + +JSON lines carrying ``job_id``, ``asset_id``, and ``operation_id`` when present, +so later stages can correlate work across the API, worker, and jobs. Secrets are +never passed to the logger; ``SecretStr`` masks them even if one slips through. +""" + +from __future__ import annotations + +import json +import logging +import sys +from datetime import datetime, timezone + +CONTEXT_FIELDS = ("job_id", "asset_id", "operation_id") + + +class JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + payload = { + "ts": datetime.fromtimestamp(record.created, timezone.utc).isoformat(), + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + } + for field in CONTEXT_FIELDS: + value = getattr(record, field, None) + if value is not None: + payload[field] = value + if record.exc_info: + payload["exc"] = self.formatException(record.exc_info) + return json.dumps(payload, default=str) + + +def configure_logging(level: str = "INFO", fmt: str = "json") -> None: + handler = logging.StreamHandler(sys.stderr) + if fmt == "json": + handler.setFormatter(JsonFormatter()) + else: + handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s")) + root = logging.getLogger() + root.handlers.clear() + root.addHandler(handler) + root.setLevel(level.upper()) diff --git a/photo_pipeline/models/__init__.py b/photo_pipeline/models/__init__.py new file mode 100644 index 0000000..7fac3c6 --- /dev/null +++ b/photo_pipeline/models/__init__.py @@ -0,0 +1,9 @@ +"""SQLAlchemy persistence models. + +Importing this package registers every model on ``Base.metadata``, which the +Alembic environment relies on. +""" + +from photo_pipeline.models.assets import Asset, AssetPath + +__all__ = ["Asset", "AssetPath"] diff --git a/photo_pipeline/models/assets.py b/photo_pipeline/models/assets.py new file mode 100644 index 0000000..53fc547 --- /dev/null +++ b/photo_pipeline/models/assets.py @@ -0,0 +1,59 @@ +"""Stable asset identity and path history. + +A file path is mutable metadata, never identity: ``assets.id`` is a generated +UUID that never changes across moves, renames, or archival. ``asset_paths`` +records every path an asset has occupied. ``state_version`` supports optimistic +concurrency; ``created_at``/``updated_at`` are the audit timestamps. + +Later-stage columns (canonical link, safety score, archive location) are added by +the migrations of the stories that own them, not speculatively here. +""" + +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import DateTime, ForeignKey, Integer, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from photo_pipeline.db import Base + + +class Asset(Base): + __tablename__ = "assets" + + id: Mapped[str] = mapped_column(String, primary_key=True) + original_path: Mapped[str] = mapped_column(String, nullable=False) + current_path: Mapped[str | None] = mapped_column(String, unique=True) + + current_sha256: Mapped[str | None] = mapped_column(String) + pixel_sha256: Mapped[str | None] = mapped_column(String) + phash: Mapped[str | None] = mapped_column(String) + hash_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + byte_size: Mapped[int | None] = mapped_column(Integer) + + discovered_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + missing_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + availability_state: Mapped[str] = mapped_column(String, nullable=False, default="active") + + state_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now() + ) + + +class AssetPath(Base): + __tablename__ = "asset_paths" + + asset_id: Mapped[str] = mapped_column( + ForeignKey("assets.id"), primary_key=True + ) + path: Mapped[str] = mapped_column(String, primary_key=True) + valid_from: Mapped[datetime] = mapped_column( + DateTime(timezone=True), primary_key=True + ) + valid_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + reason: Mapped[str | None] = mapped_column(String) diff --git a/pyproject.toml b/pyproject.toml index 1d6ea4a..7b6993e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,19 @@ name = "photoanalyzer" version = "0.1.0" requires-python = ">=3.12" +dependencies = [ + "fastapi>=0.115", + "uvicorn>=0.30", + "sqlalchemy>=2.0", + "alembic>=1.13", + "pydantic>=2.7", +] + +[project.optional-dependencies] +test = ["pytest>=8", "httpx>=0.27"] [tool.ruff] line-length = 100 + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..6340734 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,8 @@ +"""Make the repository root importable for the pipeline test suites.""" + +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +if str(REPO) not in sys.path: + sys.path.insert(0, str(REPO)) diff --git a/tests/integration/test_app_lifecycle.py b/tests/integration/test_app_lifecycle.py new file mode 100644 index 0000000..692b045 --- /dev/null +++ b/tests/integration/test_app_lifecycle.py @@ -0,0 +1,163 @@ +"""Application lifecycle: readiness, WAL, foreign keys, restart, clean shutdown. + +In-process tests drive the real ASGI app (its lifespan runs migrations, opens the +engine, and disposes it). One process-level test launches ``python -m +photo_pipeline serve`` as a real child process and waits on readiness, matching +production boundaries. +""" + +import os +import socket +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +import httpx +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import text +from sqlalchemy.exc import IntegrityError + +from photo_pipeline.api.app import create_app +from photo_pipeline.config import Config +from photo_pipeline.models import Asset, AssetPath + +REPO = Path(__file__).resolve().parents[2] + + +@pytest.fixture +def config(tmp_path): + return Config.from_env({"PHOTO_PIPELINE_DATA_DIR": str(tmp_path)}) + + +def test_live_and_ready(config): + app = create_app(config) + with TestClient(app) as client: + assert client.get("/api/v1/health/live").json() == {"status": "alive"} + ready = client.get("/api/v1/health/ready") + assert ready.status_code == 200 + body = ready.json() + assert body["status"] == "ready" + assert str(body["checks"]["journal_mode"]).lower() == "wal" + assert body["checks"]["foreign_keys"] == 1 + + +def test_ready_reports_wal_and_foreign_keys_on_engine(config): + app = create_app(config) + with TestClient(app): + with app.state.engine.connect() as conn: + assert str(conn.execute(text("PRAGMA journal_mode")).scalar()).lower() == "wal" + assert int(conn.execute(text("PRAGMA foreign_keys")).scalar()) == 1 + + +def test_foreign_keys_are_enforced(config): + app = create_app(config) + with TestClient(app): + session = app.state.session_factory() + try: + session.add( + AssetPath( + asset_id="does-not-exist", + path="ghost.jpg", + valid_from=datetime.now(timezone.utc), + ) + ) + with pytest.raises(IntegrityError): + session.commit() + finally: + session.close() + + +def test_restart_preserves_data_and_reruns_migrations(config): + app1 = create_app(config) + with TestClient(app1): + session = app1.state.session_factory() + try: + session.add( + Asset( + id="asset-1", + original_path="one.jpg", + current_path="one.jpg", + hash_version=1, + discovered_at=datetime.now(timezone.utc), + ) + ) + session.commit() + finally: + session.close() + + # Fresh app against the same data dir: migrations are idempotent, data survives. + app2 = create_app(config) + with TestClient(app2) as client: + assert client.get("/api/v1/health/ready").status_code == 200 + session = app2.state.session_factory() + try: + asset = session.get(Asset, "asset-1") + assert asset is not None + assert asset.current_path == "one.jpg" + finally: + session.close() + + +def test_shutdown_disposes_engine(config): + app = create_app(config) + with TestClient(app): + engine = app.state.engine + assert engine is not None + # Lifespan shutdown ran: state cleared and no connections left checked out. + assert app.state.engine is None + assert engine.pool.checkedout() == 0 + + +def _free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def test_process_level_readiness_and_clean_shutdown(tmp_path): + port = _free_port() + env = { + **os.environ, + "PHOTO_PIPELINE_DATA_DIR": str(tmp_path), + "PHOTO_PIPELINE_HOST": "127.0.0.1", + "PHOTO_PIPELINE_PORT": str(port), + } + proc = subprocess.Popen( + [sys.executable, "-m", "photo_pipeline", "serve"], + cwd=str(REPO), + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + base = f"http://127.0.0.1:{port}/api/v1/health/ready" + deadline = time.monotonic() + 30 + body = None + while time.monotonic() < deadline: + if proc.poll() is not None: + out, err = proc.communicate() + pytest.fail(f"server exited early: {err.decode(errors='replace')}") + try: + response = httpx.get(base, timeout=1.0) + if response.status_code == 200: + body = response.json() + break + except httpx.HTTPError: + time.sleep(0.2) + assert body is not None, "server never became ready" + assert body["status"] == "ready" + assert str(body["checks"]["journal_mode"]).lower() == "wal" + assert body["checks"]["foreign_keys"] == 1 + finally: + proc.terminate() + try: + proc.wait(timeout=15) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + pytest.fail("server did not shut down cleanly on SIGTERM") + # Clean shutdown: terminated by our signal, not a crash. + assert proc.returncode in (0, -15) diff --git a/tests/integration/test_migrations.py b/tests/integration/test_migrations.py new file mode 100644 index 0000000..c9a89db --- /dev/null +++ b/tests/integration/test_migrations.py @@ -0,0 +1,101 @@ +"""Migrations run from an empty database and from a legacy schema snapshot.""" + +import sqlite3 +from pathlib import Path + +import pytest + +from photo_pipeline.db import run_migrations + +# Faithful snapshot of the donor photo_analyzer.py schema (photos + FTS + triggers). +LEGACY_SCHEMA = """ +CREATE TABLE photos ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT UNIQUE NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + phash TEXT, file_sha1 TEXT, dup_of TEXT, + description TEXT, tags TEXT, people_count INTEGER, setting TEXT, + time_of_day TEXT, season TEXT, mood TEXT, location_hint TEXT, approx_year INTEGER, + raw_response TEXT, error_message TEXT, analyzed_at TEXT, exif_written_at TEXT +); +CREATE INDEX idx_status ON photos(status); +CREATE INDEX idx_path ON photos(path); +CREATE VIRTUAL TABLE photos_fts USING fts5( + path, description, tags, mood, location_hint, + content=photos, content_rowid=id +); +""" + + +def _tables(db: Path) -> set[str]: + conn = sqlite3.connect(db) + try: + rows = conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall() + return {r[0] for r in rows} + finally: + conn.close() + + +def _head_revision(db: Path) -> str: + conn = sqlite3.connect(db) + try: + return conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] + finally: + conn.close() + + +def test_migrations_from_empty_database(tmp_path): + db = tmp_path / "empty.db" + run_migrations(f"sqlite:///{db}") + + tables = _tables(db) + assert {"assets", "asset_paths"} <= tables + assert _head_revision(db) == "0001_initial_identity" + + +def test_migrations_from_legacy_snapshot_preserve_existing_data(tmp_path): + db = tmp_path / "legacy.db" + conn = sqlite3.connect(db) + conn.executescript(LEGACY_SCHEMA) + conn.execute( + "INSERT INTO photos (path, status, description) VALUES (?, ?, ?)", + ("album/one.jpg", "analyzed", "a red block"), + ) + conn.commit() + conn.close() + + run_migrations(f"sqlite:///{db}") + + tables = _tables(db) + assert {"photos", "assets", "asset_paths"} <= tables + + conn = sqlite3.connect(db) + try: + row = conn.execute("SELECT path, status, description FROM photos").fetchone() + finally: + conn.close() + assert row == ("album/one.jpg", "analyzed", "a red block") + assert _head_revision(db) == "0001_initial_identity" + + +def test_migrations_are_idempotent(tmp_path): + db = tmp_path / "twice.db" + url = f"sqlite:///{db}" + run_migrations(url) + run_migrations(url) # second run is a no-op at head + assert _head_revision(db) == "0001_initial_identity" + + +@pytest.mark.parametrize("expected", ["assets", "asset_paths"]) +def test_identity_tables_have_versions_and_audit_columns(tmp_path, expected): + db = tmp_path / "cols.db" + run_migrations(f"sqlite:///{db}") + conn = sqlite3.connect(db) + try: + cols = {r[1] for r in conn.execute(f"PRAGMA table_info({expected})")} + finally: + conn.close() + if expected == "assets": + assert {"id", "state_version", "created_at", "updated_at", "discovered_at"} <= cols + else: + assert {"asset_id", "path", "valid_from", "valid_until"} <= cols diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py new file mode 100644 index 0000000..5892a0e --- /dev/null +++ b/tests/unit/test_config.py @@ -0,0 +1,41 @@ +"""Config parses env and never leaks secrets.""" + +from pathlib import Path + +from photo_pipeline.config import Config + + +def test_from_env_reads_prefixed_values(): + config = Config.from_env( + { + "PHOTO_PIPELINE_DATA_DIR": "/tmp/pp", + "PHOTO_PIPELINE_PORT": "9123", + "PHOTO_PIPELINE_LOG_FORMAT": "text", + "UNRELATED": "ignored", + } + ) + assert config.data_dir == Path("/tmp/pp") + assert config.port == 9123 + assert config.log_format == "text" + + +def test_defaults_and_derived_paths(): + config = Config.from_env({}) + assert config.host == "127.0.0.1" + assert config.database_path == Path("data") / "photo_pipeline.db" + assert config.database_url == f"sqlite:///{config.database_path}" + + +def test_secrets_are_masked_everywhere_but_get_secret_value(): + config = Config.from_env({"PHOTO_PIPELINE_VISION_API_KEY": "super-secret-key"}) + assert config.vision_api_key.get_secret_value() == "super-secret-key" + # Masked in repr, str, and serialized output. + assert "super-secret-key" not in repr(config) + assert "super-secret-key" not in str(config) + assert "super-secret-key" not in config.model_dump_json() + + +def test_missing_secret_is_none(): + config = Config.from_env({}) + assert config.vision_api_key is None + assert config.immich_api_key is None diff --git a/work_item/.work-item.yml b/work_item/.work-item.yml index 1f6c9fb..f34ce0d 100644 --- a/work_item/.work-item.yml +++ b/work_item/.work-item.yml @@ -10,7 +10,7 @@ workflow: require_ci: false required_tests: - work_item/scripts/python -m unittest discover -s work_item/tests -v - - work_item/scripts/python -m pytest tests/characterization -q + - work_item/scripts/python -m pytest tests -q safety: max_file_bytes: 5000000