Add flight comparator web app with full scan pipeline

Full-stack flight price scanner built on fast-flights v3 (SOCS cookie bypass):

Backend (FastAPI + SQLite):
- REST API with rate limiting, Pydantic v2 validation, paginated responses
- Scan pipeline: resolves airports, queries every day in the window, saves
  individual flights + aggregate route stats to SQLite
- Background async scan processor with real-time progress tracking
- Airport search endpoint backed by OpenFlights dataset
- Daily scan window (all dates, not monthly samples)

Frontend (React 19 + TypeScript + Tailwind CSS v4):
- Dashboard with live scan status and recent scans
- Create scan form: country mode or specific airports (searchable dropdown)
- Scan detail page with expandable route rows showing individual flights
  (date, airline, departure, arrival, price) loaded on demand
- AirportSearch component with debounced live search and multi-select

Database:
- scans → routes → flights schema with FK cascade and auto-update triggers
- Migrations for schema evolution (relaxed country constraint)

Tests:
- 74 tests: unit + integration, isolated per-test SQLite DB
- Confirmed flight fixtures in tests/confirmed_flights.json (50 real flights,
  BDS→FMM Ryanair + BDS→DUS Eurowings, scraped Feb 2026)
- Integration tests parametrized from confirmed routes

Docker:
- Multi-stage builds, Compose orchestration, Nginx reverse proxy

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-02-26 17:11:51 +01:00
parent aea7590874
commit 6421f83ca7
67 changed files with 37173 additions and 0 deletions

View File

@@ -0,0 +1,9 @@
"""
Database package for Flight Radar Web App.
This package handles database initialization, migrations, and connections.
"""
from .init_db import initialize_database, get_connection
__all__ = ['initialize_database', 'get_connection']

View File

@@ -0,0 +1,296 @@
#!/usr/bin/env python3
"""
Database Initialization Script for Flight Radar Web App
This script initializes or migrates the SQLite database for the web app.
It extends the existing cache.db with new tables while preserving existing data.
Usage:
# As script
python database/init_db.py
# As module
from database import initialize_database
initialize_database()
"""
import os
import sqlite3
import sys
from pathlib import Path
def get_db_path():
"""Get path to cache.db in the flight-comparator directory."""
# Assuming this script is in flight-comparator/database/
script_dir = Path(__file__).parent
project_dir = script_dir.parent
db_path = project_dir / "cache.db"
return db_path
def get_schema_path():
"""Get path to schema.sql"""
return Path(__file__).parent / "schema.sql"
def load_schema():
"""Load schema.sql file content."""
schema_path = get_schema_path()
if not schema_path.exists():
raise FileNotFoundError(f"Schema file not found: {schema_path}")
with open(schema_path, 'r', encoding='utf-8') as f:
return f.read()
def check_existing_tables(conn):
"""Check which tables already exist in the database."""
cursor = conn.execute("""
SELECT name FROM sqlite_master
WHERE type='table'
ORDER BY name
""")
return [row[0] for row in cursor.fetchall()]
def get_schema_version(conn):
"""Get current schema version, or 0 if schema_version table doesn't exist."""
existing_tables = check_existing_tables(conn)
if 'schema_version' not in existing_tables:
return 0
cursor = conn.execute("SELECT MAX(version) FROM schema_version")
result = cursor.fetchone()[0]
return result if result is not None else 0
def _migrate_relax_country_constraint(conn, verbose=True):
"""
Migration: Relax the country column CHECK constraint from = 2 to >= 2.
The country column stores either a 2-letter ISO country code (e.g., 'DE')
or a comma-separated list of destination IATA codes (e.g., 'MUC,FRA,BER').
The original CHECK(length(country) = 2) only allowed country codes.
"""
cursor = conn.execute(
"SELECT sql FROM sqlite_master WHERE type='table' AND name='scans'"
)
row = cursor.fetchone()
if not row or 'length(country) = 2' not in row[0]:
return # Table doesn't exist yet or already migrated
if verbose:
print(" 🔄 Migrating scans table: relaxing country CHECK constraint...")
# SQLite doesn't support ALTER TABLE MODIFY COLUMN, so we must recreate.
# Steps:
# 1. Disable FK checks (routes has FK to scans)
# 2. Drop triggers that reference scans from routes table (they'll be
# recreated by executescript(schema_sql) below)
# 3. Create scans_new with relaxed constraint
# 4. Copy data, drop scans, rename scans_new -> scans
# 5. Re-enable FK checks
conn.execute("PRAGMA foreign_keys = OFF")
# Drop triggers on routes that reference scans in their bodies.
# They are recreated by the subsequent executescript(schema_sql) call.
conn.execute("DROP TRIGGER IF EXISTS update_scan_flight_count_insert")
conn.execute("DROP TRIGGER IF EXISTS update_scan_flight_count_update")
conn.execute("DROP TRIGGER IF EXISTS update_scan_flight_count_delete")
conn.execute("""
CREATE TABLE scans_new (
id INTEGER PRIMARY KEY AUTOINCREMENT,
origin TEXT NOT NULL CHECK(length(origin) = 3),
country TEXT NOT NULL CHECK(length(country) >= 2),
start_date TEXT NOT NULL,
end_date TEXT NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
status TEXT NOT NULL DEFAULT 'pending'
CHECK(status IN ('pending', 'running', 'completed', 'failed')),
total_routes INTEGER NOT NULL DEFAULT 0 CHECK(total_routes >= 0),
routes_scanned INTEGER NOT NULL DEFAULT 0 CHECK(routes_scanned >= 0),
total_flights INTEGER NOT NULL DEFAULT 0 CHECK(total_flights >= 0),
error_message TEXT,
seat_class TEXT DEFAULT 'economy',
adults INTEGER DEFAULT 1 CHECK(adults > 0 AND adults <= 9),
CHECK(end_date >= start_date),
CHECK(routes_scanned <= total_routes OR total_routes = 0)
)
""")
conn.execute("INSERT INTO scans_new SELECT * FROM scans")
conn.execute("DROP TABLE scans")
conn.execute("ALTER TABLE scans_new RENAME TO scans")
conn.execute("PRAGMA foreign_keys = ON")
conn.commit()
if verbose:
print(" ✅ Migration complete: country column now accepts >= 2 chars")
def initialize_database(db_path=None, verbose=True):
"""
Initialize or migrate the database.
Args:
db_path: Path to database file (default: cache.db or DATABASE_PATH env var)
verbose: Print status messages
Returns:
sqlite3.Connection: Database connection with foreign keys enabled
"""
if db_path is None:
# Check for DATABASE_PATH environment variable first (used by tests)
env_db_path = os.environ.get('DATABASE_PATH')
if env_db_path:
db_path = Path(env_db_path)
else:
db_path = get_db_path()
db_exists = db_path.exists()
if verbose:
if db_exists:
print(f"📊 Extending existing database: {db_path}")
else:
print(f"📊 Creating new database: {db_path}")
# Connect to database
conn = sqlite3.connect(db_path)
conn.row_factory = sqlite3.Row # Access columns by name
# Get existing state
existing_tables = check_existing_tables(conn)
current_version = get_schema_version(conn)
if verbose:
if existing_tables:
print(f" Existing tables: {', '.join(existing_tables)}")
print(f" Current schema version: {current_version}")
else:
print(" No existing tables found")
# Apply migrations before running schema
_migrate_relax_country_constraint(conn, verbose)
# Load and execute schema
schema_sql = load_schema()
if verbose:
print(" Executing schema...")
try:
# Execute schema (uses CREATE TABLE IF NOT EXISTS, so safe)
conn.executescript(schema_sql)
conn.commit()
if verbose:
print(" ✅ Schema executed successfully")
except sqlite3.Error as e:
conn.rollback()
if verbose:
print(f" ❌ Schema execution failed: {e}")
raise
# Verify foreign keys are enabled
cursor = conn.execute("PRAGMA foreign_keys")
fk_enabled = cursor.fetchone()[0]
if fk_enabled != 1:
if verbose:
print(" ⚠️ Warning: Foreign keys not enabled!")
print(" Enabling foreign keys for this connection...")
conn.execute("PRAGMA foreign_keys = ON")
# Check what was created
new_tables = check_existing_tables(conn)
new_version = get_schema_version(conn)
if verbose:
print(f"\n✅ Database initialized successfully!")
print(f" Total tables: {len(new_tables)}")
print(f" Schema version: {new_version}")
print(f" Foreign keys: {'✅ Enabled' if fk_enabled else '❌ Disabled'}")
# Count indexes, triggers, views
cursor = conn.execute("""
SELECT type, COUNT(*) as count
FROM sqlite_master
WHERE type IN ('index', 'trigger', 'view')
GROUP BY type
""")
for row in cursor:
print(f" {row[0].capitalize()}s: {row[1]}")
# Check for web app tables specifically
web_tables = [t for t in new_tables if t in ('scans', 'routes', 'schema_version')]
if web_tables:
print(f"\n📦 Web app tables: {', '.join(web_tables)}")
# Check for existing cache tables
cache_tables = [t for t in new_tables if 'flight' in t.lower()]
if cache_tables:
print(f"💾 Existing cache tables: {', '.join(cache_tables)}")
return conn
def get_connection(db_path=None):
"""
Get a database connection with foreign keys enabled.
Args:
db_path: Path to database file (default: cache.db or DATABASE_PATH env var)
Returns:
sqlite3.Connection: Database connection
"""
if db_path is None:
# Check for DATABASE_PATH environment variable first (used by tests)
env_db_path = os.environ.get('DATABASE_PATH')
if env_db_path:
db_path = Path(env_db_path)
else:
db_path = get_db_path()
if not db_path.exists():
raise FileNotFoundError(
f"Database not found: {db_path}\n"
"Run 'python database/init_db.py' to create it."
)
conn = sqlite3.connect(db_path)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA foreign_keys = ON")
return conn
def main():
"""CLI entry point."""
print("=" * 60)
print("Flight Radar Web App - Database Initialization")
print("=" * 60)
print()
try:
conn = initialize_database(verbose=True)
conn.close()
print()
print("=" * 60)
print("✅ Done! Database is ready.")
print("=" * 60)
return 0
except Exception as e:
print()
print("=" * 60)
print(f"❌ Error: {e}")
print("=" * 60)
return 1
if __name__ == "__main__":
sys.exit(main())

View File

@@ -0,0 +1,261 @@
-- Flight Radar Web App - Database Schema
-- Version: 2.0
-- Date: 2026-02-23
-- Database: SQLite 3
--
-- This schema extends the existing cache.db with new tables for the web app.
-- Existing tables (flight_searches, flight_results) are preserved.
-- ============================================================================
-- CRITICAL: Enable Foreign Keys (SQLite default is OFF!)
-- ============================================================================
PRAGMA foreign_keys = ON;
-- ============================================================================
-- Table: scans
-- Purpose: Track flight scan requests and their status
-- ============================================================================
CREATE TABLE IF NOT EXISTS scans (
-- Primary key with auto-increment
id INTEGER PRIMARY KEY AUTOINCREMENT,
-- Search parameters (validated by CHECK constraints)
origin TEXT NOT NULL CHECK(length(origin) = 3),
country TEXT NOT NULL CHECK(length(country) >= 2),
start_date TEXT NOT NULL, -- ISO 8601: YYYY-MM-DD
end_date TEXT NOT NULL,
-- Timestamps (auto-managed)
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-- Scan status (enforced enum via CHECK)
status TEXT NOT NULL DEFAULT 'pending'
CHECK(status IN ('pending', 'running', 'completed', 'failed')),
-- Progress tracking
total_routes INTEGER NOT NULL DEFAULT 0 CHECK(total_routes >= 0),
routes_scanned INTEGER NOT NULL DEFAULT 0 CHECK(routes_scanned >= 0),
total_flights INTEGER NOT NULL DEFAULT 0 CHECK(total_flights >= 0),
-- Error information (NULL if no error)
error_message TEXT,
-- Additional search parameters
seat_class TEXT DEFAULT 'economy',
adults INTEGER DEFAULT 1 CHECK(adults > 0 AND adults <= 9),
-- Constraints across columns
CHECK(end_date >= start_date),
CHECK(routes_scanned <= total_routes OR total_routes = 0)
);
-- Performance indexes for scans table
CREATE INDEX IF NOT EXISTS idx_scans_origin_country
ON scans(origin, country);
CREATE INDEX IF NOT EXISTS idx_scans_status
ON scans(status)
WHERE status IN ('pending', 'running'); -- Partial index for active scans
CREATE INDEX IF NOT EXISTS idx_scans_created_at
ON scans(created_at DESC); -- For recent scans query
-- ============================================================================
-- Table: routes
-- Purpose: Store discovered routes with flight statistics
-- ============================================================================
CREATE TABLE IF NOT EXISTS routes (
-- Primary key
id INTEGER PRIMARY KEY AUTOINCREMENT,
-- Foreign key to scans (cascade delete)
scan_id INTEGER NOT NULL,
-- Destination airport
destination TEXT NOT NULL CHECK(length(destination) = 3),
destination_name TEXT NOT NULL,
destination_city TEXT,
-- Flight statistics
flight_count INTEGER NOT NULL DEFAULT 0 CHECK(flight_count >= 0),
airlines TEXT NOT NULL, -- JSON array: ["Ryanair", "Lufthansa"]
-- Price statistics (NULL if no flights)
min_price REAL CHECK(min_price >= 0),
max_price REAL CHECK(max_price >= 0),
avg_price REAL CHECK(avg_price >= 0),
-- Timestamp
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-- Foreign key constraint with cascade delete
FOREIGN KEY (scan_id)
REFERENCES scans(id)
ON DELETE CASCADE,
-- Price consistency constraints
CHECK(max_price >= min_price OR max_price IS NULL),
CHECK(avg_price >= min_price OR avg_price IS NULL),
CHECK(avg_price <= max_price OR avg_price IS NULL)
);
-- Performance indexes for routes table
CREATE INDEX IF NOT EXISTS idx_routes_scan_id
ON routes(scan_id);
CREATE INDEX IF NOT EXISTS idx_routes_destination
ON routes(destination);
CREATE INDEX IF NOT EXISTS idx_routes_min_price
ON routes(min_price)
WHERE min_price IS NOT NULL; -- Partial index for routes with prices
-- ============================================================================
-- Triggers: Auto-update timestamps and aggregates
-- ============================================================================
-- Trigger: Update scans.updated_at on any update
CREATE TRIGGER IF NOT EXISTS update_scans_timestamp
AFTER UPDATE ON scans
FOR EACH ROW
BEGIN
UPDATE scans
SET updated_at = CURRENT_TIMESTAMP
WHERE id = NEW.id;
END;
-- Trigger: Update total_flights count when routes are inserted
CREATE TRIGGER IF NOT EXISTS update_scan_flight_count_insert
AFTER INSERT ON routes
FOR EACH ROW
BEGIN
UPDATE scans
SET total_flights = (
SELECT COALESCE(SUM(flight_count), 0)
FROM routes
WHERE scan_id = NEW.scan_id
)
WHERE id = NEW.scan_id;
END;
-- Trigger: Update total_flights count when routes are updated
CREATE TRIGGER IF NOT EXISTS update_scan_flight_count_update
AFTER UPDATE OF flight_count ON routes
FOR EACH ROW
BEGIN
UPDATE scans
SET total_flights = (
SELECT COALESCE(SUM(flight_count), 0)
FROM routes
WHERE scan_id = NEW.scan_id
)
WHERE id = NEW.scan_id;
END;
-- Trigger: Update total_flights count when routes are deleted
CREATE TRIGGER IF NOT EXISTS update_scan_flight_count_delete
AFTER DELETE ON routes
FOR EACH ROW
BEGIN
UPDATE scans
SET total_flights = (
SELECT COALESCE(SUM(flight_count), 0)
FROM routes
WHERE scan_id = OLD.scan_id
)
WHERE id = OLD.scan_id;
END;
-- ============================================================================
-- Table: flights
-- Purpose: Store individual flights discovered per scan
-- ============================================================================
CREATE TABLE IF NOT EXISTS flights (
id INTEGER PRIMARY KEY AUTOINCREMENT,
-- Foreign key to scans (cascade delete)
scan_id INTEGER NOT NULL,
-- Route
destination TEXT NOT NULL CHECK(length(destination) = 3),
date TEXT NOT NULL, -- ISO 8601: YYYY-MM-DD
-- Flight details
airline TEXT,
departure_time TEXT, -- HH:MM
arrival_time TEXT, -- HH:MM
price REAL CHECK(price >= 0),
stops INTEGER NOT NULL DEFAULT 0,
-- Timestamp
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (scan_id)
REFERENCES scans(id)
ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_flights_scan_id
ON flights(scan_id);
CREATE INDEX IF NOT EXISTS idx_flights_scan_dest
ON flights(scan_id, destination);
CREATE INDEX IF NOT EXISTS idx_flights_price
ON flights(scan_id, price ASC)
WHERE price IS NOT NULL;
-- ============================================================================
-- Views: Useful queries
-- ============================================================================
-- View: Recent scans with route counts
CREATE VIEW IF NOT EXISTS recent_scans AS
SELECT
s.id,
s.origin,
s.country,
s.status,
s.created_at,
s.total_routes,
s.total_flights,
COUNT(r.id) as routes_found,
MIN(r.min_price) as cheapest_flight,
s.error_message
FROM scans s
LEFT JOIN routes r ON r.scan_id = s.id
GROUP BY s.id
ORDER BY s.created_at DESC
LIMIT 10;
-- View: Active scans (pending or running)
CREATE VIEW IF NOT EXISTS active_scans AS
SELECT *
FROM scans
WHERE status IN ('pending', 'running')
ORDER BY created_at ASC;
-- ============================================================================
-- Initial Data: None (tables start empty)
-- ============================================================================
-- Schema version tracking (for future migrations)
CREATE TABLE IF NOT EXISTS schema_version (
version INTEGER PRIMARY KEY,
applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
description TEXT
);
INSERT OR IGNORE INTO schema_version (version, description)
VALUES (1, 'Initial web app schema with scans and routes tables');
-- ============================================================================
-- Verification Queries (for testing)
-- ============================================================================
-- Uncomment to verify schema creation:
-- SELECT name, type FROM sqlite_master WHERE type IN ('table', 'index', 'trigger', 'view') ORDER BY type, name;
-- PRAGMA foreign_keys;
-- PRAGMA table_info(scans);
-- PRAGMA table_info(routes);