feat: implement reverse scan (country → specific airports)
All checks were successful
Deploy / deploy (push) Successful in 30s

- DB schema: relaxed origin CHECK to >=2 chars, added scan_mode column to
  scans and scheduled_scans, added origin_airport to routes and flights,
  updated unique index to (scan_id, COALESCE(origin_airport,''), destination)
- Migrations: init_db.py recreates tables and adds columns via guarded ALTERs
- API: scan_mode field on ScanRequest/Scan; Route/Flight expose origin_airport;
  GET /scans/{id}/flights accepts origin_airport filter; CreateScheduleRequest
  and Schedule carry scan_mode; scheduler and run-now pass scan_mode through
- scan_processor: _write_route_incremental accepts origin_airport; process_scan
  branches on scan_mode=reverse (country → airports × destinations × dates)
- Frontend: new CountrySelect component (populated from GET /api/v1/countries);
  Scans page adds Direction toggle + CountrySelect for both modes; ScanDetails
  shows Origin column for reverse scans and uses composite route keys; Re-run
  preserves scan_mode

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-01 17:58:55 +01:00
parent 7ece1f9b45
commit 77d2a46264
9 changed files with 1070 additions and 279 deletions

View File

@@ -52,7 +52,7 @@ def stop_scan_task(scan_id: int) -> bool:
def _write_route_incremental(scan_id: int, destination: str,
dest_name: str, dest_city: str,
new_flights: list):
new_flights: list, origin_airport: str = None):
"""
Write or update a route row and its individual flight rows immediately.
@@ -60,6 +60,9 @@ def _write_route_incremental(scan_id: int, destination: str,
query returns results. Merges into the existing route row if one already
exists, using a running weighted average for avg_price.
For reverse scans, origin_airport is the variable origin IATA code.
For forward scans, origin_airport is None.
Opens its own DB connection — safe to call from the event loop thread.
"""
prices = [f.get('price') for f in new_flights if f.get('price')]
@@ -76,21 +79,29 @@ def _write_route_incremental(scan_id: int, destination: str,
conn = get_connection()
cursor = conn.cursor()
cursor.execute("""
SELECT id, flight_count, min_price, max_price, avg_price, airlines
FROM routes
WHERE scan_id = ? AND destination = ?
""", (scan_id, destination))
# Fetch existing route row (key: scan_id + origin_airport + destination)
if origin_airport is None:
cursor.execute("""
SELECT id, flight_count, min_price, max_price, avg_price, airlines
FROM routes
WHERE scan_id = ? AND origin_airport IS NULL AND destination = ?
""", (scan_id, destination))
else:
cursor.execute("""
SELECT id, flight_count, min_price, max_price, avg_price, airlines
FROM routes
WHERE scan_id = ? AND origin_airport = ? AND destination = ?
""", (scan_id, origin_airport, destination))
existing = cursor.fetchone()
if existing is None:
cursor.execute("""
INSERT INTO routes (
scan_id, destination, destination_name, destination_city,
scan_id, origin_airport, destination, destination_name, destination_city,
flight_count, airlines, min_price, max_price, avg_price
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (
scan_id, destination, dest_name, dest_city,
scan_id, origin_airport, destination, dest_name, dest_city,
new_count, json.dumps(new_airlines),
new_min, new_max, new_avg,
))
@@ -107,29 +118,44 @@ def _write_route_incremental(scan_id: int, destination: str,
merged_avg = (old_avg * old_count + new_avg * new_count) / merged_count
merged_airlines = json.dumps(list(set(old_airlines) | set(new_airlines)))
cursor.execute("""
UPDATE routes
SET flight_count = ?,
min_price = ?,
max_price = ?,
avg_price = ?,
airlines = ?
WHERE scan_id = ? AND destination = ?
""", (
merged_count, merged_min, merged_max, merged_avg, merged_airlines,
scan_id, destination,
))
if origin_airport is None:
cursor.execute("""
UPDATE routes
SET flight_count = ?,
min_price = ?,
max_price = ?,
avg_price = ?,
airlines = ?
WHERE scan_id = ? AND origin_airport IS NULL AND destination = ?
""", (
merged_count, merged_min, merged_max, merged_avg, merged_airlines,
scan_id, destination,
))
else:
cursor.execute("""
UPDATE routes
SET flight_count = ?,
min_price = ?,
max_price = ?,
avg_price = ?,
airlines = ?
WHERE scan_id = ? AND origin_airport = ? AND destination = ?
""", (
merged_count, merged_min, merged_max, merged_avg, merged_airlines,
scan_id, origin_airport, destination,
))
for flight in new_flights:
if not flight.get('price'):
continue
cursor.execute("""
INSERT INTO flights (
scan_id, destination, date, airline,
scan_id, origin_airport, destination, date, airline,
departure_time, arrival_time, price, stops
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (
scan_id,
origin_airport,
destination,
flight.get('date', ''),
flight.get('airline'),
@@ -170,7 +196,7 @@ async def process_scan(scan_id: int):
cursor = conn.cursor()
cursor.execute("""
SELECT origin, country, start_date, end_date, seat_class, adults
SELECT origin, country, scan_mode, start_date, end_date, seat_class, adults
FROM scans
WHERE id = ?
""", (scan_id,))
@@ -180,9 +206,10 @@ async def process_scan(scan_id: int):
logger.error(f"[Scan {scan_id}] Scan not found in database")
return
origin, country_or_airports, start_date_str, end_date_str, seat_class, adults = row
origin, country_or_airports, scan_mode, start_date_str, end_date_str, seat_class, adults = row
scan_mode = scan_mode or 'forward'
logger.info(f"[Scan {scan_id}] Scan details: {origin} -> {country_or_airports}, {start_date_str} to {end_date_str}")
logger.info(f"[Scan {scan_id}] Scan details: mode={scan_mode}, {origin} -> {country_or_airports}, {start_date_str} to {end_date_str}")
# Update status to 'running' and record when processing started
cursor.execute("""
@@ -192,27 +219,39 @@ async def process_scan(scan_id: int):
""", (scan_id,))
conn.commit()
# Determine mode: country (2 letters) or specific airports (comma-separated)
# Resolve airports based on scan_mode
try:
if len(country_or_airports) == 2 and country_or_airports.isalpha():
# Country mode: resolve airports from country code
logger.info(f"[Scan {scan_id}] Mode: Country search ({country_or_airports})")
destinations = get_airports_for_country(country_or_airports)
if not destinations:
raise ValueError(f"No airports found for country: {country_or_airports}")
if scan_mode == 'reverse':
# Reverse scan: origin = ISO country, country_or_airports = comma-separated dest IATAs
logger.info(f"[Scan {scan_id}] Mode: Reverse scan ({origin} country → {country_or_airports})")
origin_airports = get_airports_for_country(origin)
if not origin_airports:
raise ValueError(f"No airports found for origin country: {origin}")
origin_iatas = [a['iata'] for a in origin_airports]
destination_codes = [d['iata'] for d in destinations]
logger.info(f"[Scan {scan_id}] Found {len(destination_codes)} destination airports: {destination_codes}")
destination_codes = [code.strip() for code in country_or_airports.split(',')]
dest_infos = {
code: lookup_airport(code) or {'iata': code, 'name': code, 'city': ''}
for code in destination_codes
}
logger.info(f"[Scan {scan_id}] {len(origin_iatas)} origins × {len(destination_codes)} destinations")
else:
# Specific airports mode: parse comma-separated list
destination_codes = [code.strip() for code in country_or_airports.split(',')]
destinations = [
lookup_airport(code) or {'iata': code, 'name': code, 'city': ''}
for code in destination_codes
]
logger.info(f"[Scan {scan_id}] Mode: Specific airports ({len(destination_codes)} destinations: {destination_codes})")
# Forward scan: origin = fixed IATA, country_or_airports = country code or dest IATAs
if len(country_or_airports) == 2 and country_or_airports.isalpha():
logger.info(f"[Scan {scan_id}] Mode: Forward country search ({country_or_airports})")
dest_list = get_airports_for_country(country_or_airports)
if not dest_list:
raise ValueError(f"No airports found for country: {country_or_airports}")
destination_codes = [d['iata'] for d in dest_list]
dest_infos = {d['iata']: d for d in dest_list}
else:
destination_codes = [code.strip() for code in country_or_airports.split(',')]
dest_infos = {
code: lookup_airport(code) or {'iata': code, 'name': code, 'city': ''}
for code in destination_codes
}
logger.info(f"[Scan {scan_id}] Mode: Forward specific airports ({destination_codes})")
except Exception as e:
logger.error(f"[Scan {scan_id}] Failed to resolve airports: {str(e)}")
@@ -227,8 +266,6 @@ async def process_scan(scan_id: int):
conn.commit()
return
# Note: Don't update total_routes yet - we'll set it after we know the actual number of route queries
# Generate dates to scan — every day in the window
start_date = datetime.strptime(start_date_str, '%Y-%m-%d').date()
end_date = datetime.strptime(end_date_str, '%Y-%m-%d').date()
@@ -241,11 +278,17 @@ async def process_scan(scan_id: int):
logger.info(f"[Scan {scan_id}] Will scan {len(dates)} dates: {dates}")
# Build routes list: [(origin, destination, date), ...]
# Build routes list: [(origin_iata, destination_iata, date), ...]
routes_to_scan = []
for dest in destination_codes:
for scan_date in dates:
routes_to_scan.append((origin, dest, scan_date))
if scan_mode == 'reverse':
for orig_iata in origin_iatas:
for dest_code in destination_codes:
for scan_date in dates:
routes_to_scan.append((orig_iata, dest_code, scan_date))
else:
for dest_code in destination_codes:
for scan_date in dates:
routes_to_scan.append((origin, dest_code, scan_date))
logger.info(f"[Scan {scan_id}] Total route queries: {len(routes_to_scan)}")
@@ -262,7 +305,7 @@ async def process_scan(scan_id: int):
# Signature: callback(origin, destination, date, status, count, error=None, flights=None)
routes_scanned_count = 0
def progress_callback(origin: str, destination: str, date: str,
def progress_callback(cb_origin: str, destination: str, date: str,
status: str, count: int, error: str = None,
flights: list = None):
nonlocal routes_scanned_count
@@ -274,10 +317,15 @@ async def process_scan(scan_id: int):
if flights and status in ('cache_hit', 'api_success'):
for f in flights:
f['date'] = date
dest_info = next((d for d in destinations if d['iata'] == destination), None)
dest_name = dest_info.get('name', destination) if dest_info else destination
dest_city = dest_info.get('city', '') if dest_info else ''
_write_route_incremental(scan_id, destination, dest_name, dest_city, flights)
dest_info = dest_infos.get(destination) or {'iata': destination, 'name': destination, 'city': ''}
dest_name = dest_info.get('name', destination)
dest_city = dest_info.get('city', '')
# For reverse scans, cb_origin is the variable origin airport IATA
route_origin = cb_origin if scan_mode == 'reverse' else None
_write_route_incremental(
scan_id, destination, dest_name, dest_city, flights,
origin_airport=route_origin
)
# Update progress counter
try:
@@ -295,7 +343,7 @@ async def process_scan(scan_id: int):
progress_conn.close()
if routes_scanned_count % 10 == 0:
logger.info(f"[Scan {scan_id}] Progress: {routes_scanned_count}/{len(routes_to_scan)} routes ({status}: {origin}{destination})")
logger.info(f"[Scan {scan_id}] Progress: {routes_scanned_count}/{len(routes_to_scan)} routes ({status}: {cb_origin}{destination})")
except Exception as e:
logger.error(f"[Scan {scan_id}] Failed to update progress: {str(e)}")