# FlightMonitor/controller/aircraft_db_importer.py """ Handles the import of aircraft database from CSV files, including progress tracking and multi-threading for the import process. """ import os import csv import threading import time from typing import Optional, Any, Callable, Tuple, TYPE_CHECKING from flightmonitor.utils.logger import get_logger # Type checking imports to avoid circular dependencies at runtime if TYPE_CHECKING: from flightmonitor.data.aircraft_database_manager import AircraftDatabaseManager from flightmonitor.gui.main_window import MainWindow from flightmonitor.gui.dialogs.import_progress_dialog import ImportProgressDialog module_logger = get_logger(__name__) # Constants for CSV processing CSV_FIELD_SIZE_LIMIT_BYTES = 10 * 1024 * 1024 # 10 MB PROGRESS_CALLBACK_INTERVAL_ROWS = 500 # How often to update the progress bar class AircraftDBImporter: """ Manages the process of importing aircraft data from a CSV file into the database. Designed to run the import in a separate thread and report progress to a GUI dialog. """ def __init__( self, aircraft_db_manager: "AircraftDatabaseManager", main_window: "MainWindow", ): """ Initializes the AircraftDBImporter. Args: aircraft_db_manager: An instance of AircraftDatabaseManager to perform DB operations. main_window: A reference to the main application window (for GUI updates and messages). """ self.aircraft_db_manager = aircraft_db_manager self.main_window = main_window module_logger.debug("AircraftDBImporter initialized.") def import_aircraft_database_with_progress( self, csv_filepath: str, progress_dialog_ref: "ImportProgressDialog" ): """ Starts the aircraft database import process in a separate thread. This method is intended to be called from the GUI thread. Args: csv_filepath: The path to the CSV file to import. progress_dialog_ref: A reference to the ImportProgressDialog instance for updates. """ module_logger.info( f"AircraftDBImporter: Requesting aircraft DB import with progress from: {csv_filepath}" ) # Basic validation checks before starting the thread if not self.aircraft_db_manager: module_logger.error( "AircraftDBImporter: AircraftDatabaseManager not initialized. Cannot import." ) if progress_dialog_ref and progress_dialog_ref.winfo_exists(): # Schedule update to run on the GUI thread self._schedule_gui_update( progress_dialog_ref.import_finished, False, "Error: Database manager not active.", ) elif self.main_window and hasattr(self.main_window, "show_error_message"): self.main_window.show_error_message( "Database Error", "Aircraft database manager not active." ) return if not (self.main_window and self.main_window.root.winfo_exists()): module_logger.error( "AircraftDBImporter: MainWindow not available to start import thread." ) if progress_dialog_ref and progress_dialog_ref.winfo_exists(): self._schedule_gui_update( progress_dialog_ref.import_finished, False, "Error: Main application window not available.", ) return # Notify the dialog that import has started if progress_dialog_ref and progress_dialog_ref.winfo_exists(): self._schedule_gui_update(progress_dialog_ref.import_started) # Start the import in a new daemon thread import_thread = threading.Thread( target=self._perform_db_import_threaded, args=(csv_filepath, progress_dialog_ref), daemon=True, # Daemon threads exit when the main program exits name="AircraftDBImportWorker", ) import_thread.start() module_logger.info( f"AircraftDBImporter: Import thread started for: {csv_filepath}" ) def _perform_db_import_threaded( self, csv_filepath: str, progress_dialog_ref: "ImportProgressDialog" ): """ Executes the database import process in a separate thread. This method should NOT be called directly from the GUI thread. """ if not self.aircraft_db_manager: module_logger.error( "AircraftDBImporter: AircraftDBManager N/A in import thread." ) if progress_dialog_ref and progress_dialog_ref.winfo_exists(): self._schedule_gui_update( progress_dialog_ref.import_finished, False, "Internal Error: DB Manager missing.", ) return # Attempt to increase CSV field size limit to handle large fields self._set_csv_field_size_limit() module_logger.info( f"AircraftDBImporter: Import thread: Starting row count for: {csv_filepath}" ) # Count total rows for progress bar calculation total_data_rows = self._count_csv_rows(csv_filepath) if total_data_rows is None: module_logger.error( f"AircraftDBImporter: Could not count rows in CSV: {csv_filepath}" ) if progress_dialog_ref and progress_dialog_ref.winfo_exists(): self._schedule_gui_update( progress_dialog_ref.import_finished, False, f"Error: Could not read/count rows in '{os.path.basename(csv_filepath)}'. Check logs.", ) return if total_data_rows == 0: module_logger.info( f"AircraftDBImporter: CSV file '{csv_filepath}' is empty or header-only." ) if progress_dialog_ref and progress_dialog_ref.winfo_exists(): self._schedule_gui_update( progress_dialog_ref.update_progress, 0, 0, 0, f"File '{os.path.basename(csv_filepath)}' is empty or header-only.", ) self._schedule_gui_update( progress_dialog_ref.import_finished, True, "Import complete: No data rows to import.", ) return # Initial progress update with total rows found if progress_dialog_ref and progress_dialog_ref.winfo_exists(): self._schedule_gui_update( progress_dialog_ref.update_progress, 0, 0, total_data_rows, f"Found {total_data_rows} data rows. Starting import from '{os.path.basename(csv_filepath)}'...", ) # Define the progress callback function that will be passed to the DB manager def import_progress_update_callback( processed_csv_rows, imported_db_rows, total_for_cb ): if progress_dialog_ref and progress_dialog_ref.winfo_exists(): # Schedule the GUI update on the main Tkinter thread self._schedule_gui_update( progress_dialog_ref.update_progress, processed_csv_rows, imported_db_rows, total_for_cb, f"Importing CSV row {processed_csv_rows}...", ) # Perform the actual import via AircraftDatabaseManager processed_final, imported_final = self.aircraft_db_manager.import_from_csv( csv_filepath, replace_existing=True, # Always replace existing for full import progress_callback=import_progress_update_callback, total_rows_for_callback=total_data_rows, ) # Determine final message and success status success = True final_message = f"DB Import complete. Processed: {processed_final}, Imported/Updated: {imported_final}." if imported_final == 0 and processed_final > 0 and total_data_rows > 0: success = False # Consider it a failure if no rows were imported final_message = f"DB Import: {processed_final} CSV data rows processed, 0 imported (check CSV format/logs)." module_logger.error(f"AircraftDBImporter: {final_message}") elif imported_final == 0 and processed_final == 0 and total_data_rows > 0: success = False final_message = f"DB Import failed. Could not process rows from '{os.path.basename(csv_filepath)}'." module_logger.critical(f"AircraftDBImporter: {final_message}") # Final update to the progress dialog if progress_dialog_ref and progress_dialog_ref.winfo_exists(): self._schedule_gui_update( progress_dialog_ref.update_progress, processed_final, imported_final, total_data_rows, "Finalizing import...", ) self._schedule_gui_update( progress_dialog_ref.import_finished, success, final_message ) module_logger.info( f"AircraftDBImporter: Import thread finished. Result: {final_message}" ) def _count_csv_rows(self, csv_filepath: str) -> Optional[int]: """ Counts the number of data rows in a CSV file (excluding header). Args: csv_filepath: The path to the CSV file. Returns: The number of data rows, or None if the file cannot be read or is not found. """ # Temporarily increase CSV field size limit for robust row counting self._set_csv_field_size_limit() try: with open(csv_filepath, "r", encoding="utf-8-sig") as f: reader = csv.reader(f) try: next(reader) # Skip header row except StopIteration: return 0 # File is empty or only contains header row_count = sum(1 for _ in reader) module_logger.debug( f"AircraftDBImporter: Counted {row_count} data rows in {csv_filepath}" ) return row_count except FileNotFoundError: module_logger.error( f"AircraftDBImporter: CSV file not found: {csv_filepath}" ) return None except csv.Error as e_csv: module_logger.error( f"AircraftDBImporter: CSV format error during row count of {csv_filepath}: {e_csv}", exc_info=False, ) return None except Exception as e: module_logger.error( f"AircraftDBImporter: Unexpected error counting rows in {csv_filepath}: {e}", exc_info=True, ) return None def _set_csv_field_size_limit(self): """ Attempts to increase the CSV field size limit to handle potentially large fields. """ try: current_limit = csv.field_size_limit() if CSV_FIELD_SIZE_LIMIT_BYTES > current_limit: csv.field_size_limit(CSV_FIELD_SIZE_LIMIT_BYTES) module_logger.debug( f"AircraftDBImporter: Increased csv.field_size_limit from {current_limit} to {CSV_FIELD_SIZE_LIMIT_BYTES}." ) except Exception as e: module_logger.warning( f"AircraftDBImporter: Error attempting to increase csv.field_size_limit: {e}. Proceeding with default limit.", exc_info=False, ) def _schedule_gui_update(self, callable_func: Callable, *args: Any): """ Schedules a callable function to be executed on the main Tkinter thread. This is crucial for safely updating GUI elements from a worker thread. Args: callable_func: The function to call on the GUI thread. *args: Arguments to pass to the callable function. """ if ( self.main_window and hasattr(self.main_window, "root") and self.main_window.root.winfo_exists() ): try: self.main_window.root.after(0, lambda: callable_func(*args)) except Exception as e: module_logger.error( f"AircraftDBImporter: Error scheduling GUI update from import thread: {e}", exc_info=True, ) else: module_logger.warning( f"AircraftDBImporter: Main window or root destroyed, cannot schedule GUI update for {callable_func.__name__}." )