feat: implement reverse scan (country → specific airports)
All checks were successful
Deploy / deploy (push) Successful in 30s
All checks were successful
Deploy / deploy (push) Successful in 30s
- DB schema: relaxed origin CHECK to >=2 chars, added scan_mode column to
scans and scheduled_scans, added origin_airport to routes and flights,
updated unique index to (scan_id, COALESCE(origin_airport,''), destination)
- Migrations: init_db.py recreates tables and adds columns via guarded ALTERs
- API: scan_mode field on ScanRequest/Scan; Route/Flight expose origin_airport;
GET /scans/{id}/flights accepts origin_airport filter; CreateScheduleRequest
and Schedule carry scan_mode; scheduler and run-now pass scan_mode through
- scan_processor: _write_route_incremental accepts origin_airport; process_scan
branches on scan_mode=reverse (country → airports × destinations × dates)
- Frontend: new CountrySelect component (populated from GET /api/v1/countries);
Scans page adds Direction toggle + CountrySelect for both modes; ScanDetails
shows Origin column for reverse scans and uses composite route keys; Re-run
preserves scan_mode
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -52,7 +52,7 @@ def stop_scan_task(scan_id: int) -> bool:
|
||||
|
||||
def _write_route_incremental(scan_id: int, destination: str,
|
||||
dest_name: str, dest_city: str,
|
||||
new_flights: list):
|
||||
new_flights: list, origin_airport: str = None):
|
||||
"""
|
||||
Write or update a route row and its individual flight rows immediately.
|
||||
|
||||
@@ -60,6 +60,9 @@ def _write_route_incremental(scan_id: int, destination: str,
|
||||
query returns results. Merges into the existing route row if one already
|
||||
exists, using a running weighted average for avg_price.
|
||||
|
||||
For reverse scans, origin_airport is the variable origin IATA code.
|
||||
For forward scans, origin_airport is None.
|
||||
|
||||
Opens its own DB connection — safe to call from the event loop thread.
|
||||
"""
|
||||
prices = [f.get('price') for f in new_flights if f.get('price')]
|
||||
@@ -76,21 +79,29 @@ def _write_route_incremental(scan_id: int, destination: str,
|
||||
conn = get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("""
|
||||
SELECT id, flight_count, min_price, max_price, avg_price, airlines
|
||||
FROM routes
|
||||
WHERE scan_id = ? AND destination = ?
|
||||
""", (scan_id, destination))
|
||||
# Fetch existing route row (key: scan_id + origin_airport + destination)
|
||||
if origin_airport is None:
|
||||
cursor.execute("""
|
||||
SELECT id, flight_count, min_price, max_price, avg_price, airlines
|
||||
FROM routes
|
||||
WHERE scan_id = ? AND origin_airport IS NULL AND destination = ?
|
||||
""", (scan_id, destination))
|
||||
else:
|
||||
cursor.execute("""
|
||||
SELECT id, flight_count, min_price, max_price, avg_price, airlines
|
||||
FROM routes
|
||||
WHERE scan_id = ? AND origin_airport = ? AND destination = ?
|
||||
""", (scan_id, origin_airport, destination))
|
||||
existing = cursor.fetchone()
|
||||
|
||||
if existing is None:
|
||||
cursor.execute("""
|
||||
INSERT INTO routes (
|
||||
scan_id, destination, destination_name, destination_city,
|
||||
scan_id, origin_airport, destination, destination_name, destination_city,
|
||||
flight_count, airlines, min_price, max_price, avg_price
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
scan_id, destination, dest_name, dest_city,
|
||||
scan_id, origin_airport, destination, dest_name, dest_city,
|
||||
new_count, json.dumps(new_airlines),
|
||||
new_min, new_max, new_avg,
|
||||
))
|
||||
@@ -107,29 +118,44 @@ def _write_route_incremental(scan_id: int, destination: str,
|
||||
merged_avg = (old_avg * old_count + new_avg * new_count) / merged_count
|
||||
merged_airlines = json.dumps(list(set(old_airlines) | set(new_airlines)))
|
||||
|
||||
cursor.execute("""
|
||||
UPDATE routes
|
||||
SET flight_count = ?,
|
||||
min_price = ?,
|
||||
max_price = ?,
|
||||
avg_price = ?,
|
||||
airlines = ?
|
||||
WHERE scan_id = ? AND destination = ?
|
||||
""", (
|
||||
merged_count, merged_min, merged_max, merged_avg, merged_airlines,
|
||||
scan_id, destination,
|
||||
))
|
||||
if origin_airport is None:
|
||||
cursor.execute("""
|
||||
UPDATE routes
|
||||
SET flight_count = ?,
|
||||
min_price = ?,
|
||||
max_price = ?,
|
||||
avg_price = ?,
|
||||
airlines = ?
|
||||
WHERE scan_id = ? AND origin_airport IS NULL AND destination = ?
|
||||
""", (
|
||||
merged_count, merged_min, merged_max, merged_avg, merged_airlines,
|
||||
scan_id, destination,
|
||||
))
|
||||
else:
|
||||
cursor.execute("""
|
||||
UPDATE routes
|
||||
SET flight_count = ?,
|
||||
min_price = ?,
|
||||
max_price = ?,
|
||||
avg_price = ?,
|
||||
airlines = ?
|
||||
WHERE scan_id = ? AND origin_airport = ? AND destination = ?
|
||||
""", (
|
||||
merged_count, merged_min, merged_max, merged_avg, merged_airlines,
|
||||
scan_id, origin_airport, destination,
|
||||
))
|
||||
|
||||
for flight in new_flights:
|
||||
if not flight.get('price'):
|
||||
continue
|
||||
cursor.execute("""
|
||||
INSERT INTO flights (
|
||||
scan_id, destination, date, airline,
|
||||
scan_id, origin_airport, destination, date, airline,
|
||||
departure_time, arrival_time, price, stops
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
scan_id,
|
||||
origin_airport,
|
||||
destination,
|
||||
flight.get('date', ''),
|
||||
flight.get('airline'),
|
||||
@@ -170,7 +196,7 @@ async def process_scan(scan_id: int):
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("""
|
||||
SELECT origin, country, start_date, end_date, seat_class, adults
|
||||
SELECT origin, country, scan_mode, start_date, end_date, seat_class, adults
|
||||
FROM scans
|
||||
WHERE id = ?
|
||||
""", (scan_id,))
|
||||
@@ -180,9 +206,10 @@ async def process_scan(scan_id: int):
|
||||
logger.error(f"[Scan {scan_id}] Scan not found in database")
|
||||
return
|
||||
|
||||
origin, country_or_airports, start_date_str, end_date_str, seat_class, adults = row
|
||||
origin, country_or_airports, scan_mode, start_date_str, end_date_str, seat_class, adults = row
|
||||
scan_mode = scan_mode or 'forward'
|
||||
|
||||
logger.info(f"[Scan {scan_id}] Scan details: {origin} -> {country_or_airports}, {start_date_str} to {end_date_str}")
|
||||
logger.info(f"[Scan {scan_id}] Scan details: mode={scan_mode}, {origin} -> {country_or_airports}, {start_date_str} to {end_date_str}")
|
||||
|
||||
# Update status to 'running' and record when processing started
|
||||
cursor.execute("""
|
||||
@@ -192,27 +219,39 @@ async def process_scan(scan_id: int):
|
||||
""", (scan_id,))
|
||||
conn.commit()
|
||||
|
||||
# Determine mode: country (2 letters) or specific airports (comma-separated)
|
||||
# Resolve airports based on scan_mode
|
||||
try:
|
||||
if len(country_or_airports) == 2 and country_or_airports.isalpha():
|
||||
# Country mode: resolve airports from country code
|
||||
logger.info(f"[Scan {scan_id}] Mode: Country search ({country_or_airports})")
|
||||
destinations = get_airports_for_country(country_or_airports)
|
||||
if not destinations:
|
||||
raise ValueError(f"No airports found for country: {country_or_airports}")
|
||||
if scan_mode == 'reverse':
|
||||
# Reverse scan: origin = ISO country, country_or_airports = comma-separated dest IATAs
|
||||
logger.info(f"[Scan {scan_id}] Mode: Reverse scan ({origin} country → {country_or_airports})")
|
||||
origin_airports = get_airports_for_country(origin)
|
||||
if not origin_airports:
|
||||
raise ValueError(f"No airports found for origin country: {origin}")
|
||||
origin_iatas = [a['iata'] for a in origin_airports]
|
||||
|
||||
destination_codes = [d['iata'] for d in destinations]
|
||||
|
||||
logger.info(f"[Scan {scan_id}] Found {len(destination_codes)} destination airports: {destination_codes}")
|
||||
destination_codes = [code.strip() for code in country_or_airports.split(',')]
|
||||
dest_infos = {
|
||||
code: lookup_airport(code) or {'iata': code, 'name': code, 'city': ''}
|
||||
for code in destination_codes
|
||||
}
|
||||
logger.info(f"[Scan {scan_id}] {len(origin_iatas)} origins × {len(destination_codes)} destinations")
|
||||
|
||||
else:
|
||||
# Specific airports mode: parse comma-separated list
|
||||
destination_codes = [code.strip() for code in country_or_airports.split(',')]
|
||||
destinations = [
|
||||
lookup_airport(code) or {'iata': code, 'name': code, 'city': ''}
|
||||
for code in destination_codes
|
||||
]
|
||||
logger.info(f"[Scan {scan_id}] Mode: Specific airports ({len(destination_codes)} destinations: {destination_codes})")
|
||||
# Forward scan: origin = fixed IATA, country_or_airports = country code or dest IATAs
|
||||
if len(country_or_airports) == 2 and country_or_airports.isalpha():
|
||||
logger.info(f"[Scan {scan_id}] Mode: Forward country search ({country_or_airports})")
|
||||
dest_list = get_airports_for_country(country_or_airports)
|
||||
if not dest_list:
|
||||
raise ValueError(f"No airports found for country: {country_or_airports}")
|
||||
destination_codes = [d['iata'] for d in dest_list]
|
||||
dest_infos = {d['iata']: d for d in dest_list}
|
||||
else:
|
||||
destination_codes = [code.strip() for code in country_or_airports.split(',')]
|
||||
dest_infos = {
|
||||
code: lookup_airport(code) or {'iata': code, 'name': code, 'city': ''}
|
||||
for code in destination_codes
|
||||
}
|
||||
logger.info(f"[Scan {scan_id}] Mode: Forward specific airports ({destination_codes})")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[Scan {scan_id}] Failed to resolve airports: {str(e)}")
|
||||
@@ -227,8 +266,6 @@ async def process_scan(scan_id: int):
|
||||
conn.commit()
|
||||
return
|
||||
|
||||
# Note: Don't update total_routes yet - we'll set it after we know the actual number of route queries
|
||||
|
||||
# Generate dates to scan — every day in the window
|
||||
start_date = datetime.strptime(start_date_str, '%Y-%m-%d').date()
|
||||
end_date = datetime.strptime(end_date_str, '%Y-%m-%d').date()
|
||||
@@ -241,11 +278,17 @@ async def process_scan(scan_id: int):
|
||||
|
||||
logger.info(f"[Scan {scan_id}] Will scan {len(dates)} dates: {dates}")
|
||||
|
||||
# Build routes list: [(origin, destination, date), ...]
|
||||
# Build routes list: [(origin_iata, destination_iata, date), ...]
|
||||
routes_to_scan = []
|
||||
for dest in destination_codes:
|
||||
for scan_date in dates:
|
||||
routes_to_scan.append((origin, dest, scan_date))
|
||||
if scan_mode == 'reverse':
|
||||
for orig_iata in origin_iatas:
|
||||
for dest_code in destination_codes:
|
||||
for scan_date in dates:
|
||||
routes_to_scan.append((orig_iata, dest_code, scan_date))
|
||||
else:
|
||||
for dest_code in destination_codes:
|
||||
for scan_date in dates:
|
||||
routes_to_scan.append((origin, dest_code, scan_date))
|
||||
|
||||
logger.info(f"[Scan {scan_id}] Total route queries: {len(routes_to_scan)}")
|
||||
|
||||
@@ -262,7 +305,7 @@ async def process_scan(scan_id: int):
|
||||
# Signature: callback(origin, destination, date, status, count, error=None, flights=None)
|
||||
routes_scanned_count = 0
|
||||
|
||||
def progress_callback(origin: str, destination: str, date: str,
|
||||
def progress_callback(cb_origin: str, destination: str, date: str,
|
||||
status: str, count: int, error: str = None,
|
||||
flights: list = None):
|
||||
nonlocal routes_scanned_count
|
||||
@@ -274,10 +317,15 @@ async def process_scan(scan_id: int):
|
||||
if flights and status in ('cache_hit', 'api_success'):
|
||||
for f in flights:
|
||||
f['date'] = date
|
||||
dest_info = next((d for d in destinations if d['iata'] == destination), None)
|
||||
dest_name = dest_info.get('name', destination) if dest_info else destination
|
||||
dest_city = dest_info.get('city', '') if dest_info else ''
|
||||
_write_route_incremental(scan_id, destination, dest_name, dest_city, flights)
|
||||
dest_info = dest_infos.get(destination) or {'iata': destination, 'name': destination, 'city': ''}
|
||||
dest_name = dest_info.get('name', destination)
|
||||
dest_city = dest_info.get('city', '')
|
||||
# For reverse scans, cb_origin is the variable origin airport IATA
|
||||
route_origin = cb_origin if scan_mode == 'reverse' else None
|
||||
_write_route_incremental(
|
||||
scan_id, destination, dest_name, dest_city, flights,
|
||||
origin_airport=route_origin
|
||||
)
|
||||
|
||||
# Update progress counter
|
||||
try:
|
||||
@@ -295,7 +343,7 @@ async def process_scan(scan_id: int):
|
||||
progress_conn.close()
|
||||
|
||||
if routes_scanned_count % 10 == 0:
|
||||
logger.info(f"[Scan {scan_id}] Progress: {routes_scanned_count}/{len(routes_to_scan)} routes ({status}: {origin}→{destination})")
|
||||
logger.info(f"[Scan {scan_id}] Progress: {routes_scanned_count}/{len(routes_to_scan)} routes ({status}: {cb_origin}→{destination})")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[Scan {scan_id}] Failed to update progress: {str(e)}")
|
||||
|
||||
Reference in New Issue
Block a user