feat: write routes live during scan instead of bulk-insert at completion
Routes and individual flights are now written to the database as each query result arrives, rather than after all queries finish. The frontend already polls /scans/:id/routes while status=running, so routes appear progressively with no frontend changes needed. Changes: - database/schema.sql: UNIQUE INDEX uq_routes_scan_dest(scan_id, destination) - database/init_db.py: _migrate_add_routes_unique_index() migration - scan_processor.py: _write_route_incremental() helper; progress_callback now writes routes/flights immediately; Phase 2 bulk-write replaced with a lightweight totals query - searcher_v3.py: pass flights= kwarg to progress_callback on cache_hit and api_success paths Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -130,6 +130,43 @@ def _migrate_relax_country_constraint(conn, verbose=True):
|
||||
print(" ✅ Migration complete: country column now accepts >= 2 chars")
|
||||
|
||||
|
||||
def _migrate_add_routes_unique_index(conn, verbose=True):
|
||||
"""
|
||||
Migration: Add UNIQUE index on routes(scan_id, destination).
|
||||
|
||||
Required for incremental route writes during active scans.
|
||||
Collapses any pre-existing duplicate (scan_id, destination) rows first
|
||||
(keeps the row with the lowest id) before creating the index.
|
||||
"""
|
||||
cursor = conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='index' AND name='uq_routes_scan_dest'"
|
||||
)
|
||||
if cursor.fetchone():
|
||||
return # Already migrated
|
||||
|
||||
if verbose:
|
||||
print(" 🔄 Migrating routes table: adding UNIQUE index on (scan_id, destination)...")
|
||||
|
||||
# Collapse any existing duplicates (guard against edge cases)
|
||||
conn.execute("""
|
||||
DELETE FROM routes
|
||||
WHERE id NOT IN (
|
||||
SELECT MIN(id)
|
||||
FROM routes
|
||||
GROUP BY scan_id, destination
|
||||
)
|
||||
""")
|
||||
|
||||
conn.execute("""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_routes_scan_dest
|
||||
ON routes(scan_id, destination)
|
||||
""")
|
||||
conn.commit()
|
||||
|
||||
if verbose:
|
||||
print(" ✅ Migration complete: uq_routes_scan_dest index created")
|
||||
|
||||
|
||||
def initialize_database(db_path=None, verbose=True):
|
||||
"""
|
||||
Initialize or migrate the database.
|
||||
@@ -174,6 +211,7 @@ def initialize_database(db_path=None, verbose=True):
|
||||
|
||||
# Apply migrations before running schema
|
||||
_migrate_relax_country_constraint(conn, verbose)
|
||||
_migrate_add_routes_unique_index(conn, verbose)
|
||||
|
||||
# Load and execute schema
|
||||
schema_sql = load_schema()
|
||||
|
||||
Reference in New Issue
Block a user