314 lines
13 KiB
Python
314 lines
13 KiB
Python
# FlightMonitor/controller/aircraft_db_importer.py
|
|
"""
|
|
Handles the import of aircraft database from CSV files,
|
|
including progress tracking and multi-threading for the import process.
|
|
"""
|
|
import os
|
|
import csv
|
|
import threading
|
|
import time
|
|
from typing import Optional, Any, Callable, Tuple, TYPE_CHECKING
|
|
|
|
from flightmonitor.utils.logger import get_logger
|
|
|
|
# Type checking imports to avoid circular dependencies at runtime
|
|
if TYPE_CHECKING:
|
|
from flightmonitor.data.aircraft_database_manager import AircraftDatabaseManager
|
|
from flightmonitor.gui.main_window import MainWindow
|
|
from flightmonitor.gui.dialogs.import_progress_dialog import ImportProgressDialog
|
|
|
|
module_logger = get_logger(__name__)
|
|
|
|
# Constants for CSV processing
|
|
CSV_FIELD_SIZE_LIMIT_BYTES = 10 * 1024 * 1024 # 10 MB
|
|
PROGRESS_CALLBACK_INTERVAL_ROWS = 500 # How often to update the progress bar
|
|
|
|
|
|
class AircraftDBImporter:
|
|
"""
|
|
Manages the process of importing aircraft data from a CSV file into the database.
|
|
Designed to run the import in a separate thread and report progress to a GUI dialog.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
aircraft_db_manager: "AircraftDatabaseManager",
|
|
main_window: "MainWindow",
|
|
):
|
|
"""
|
|
Initializes the AircraftDBImporter.
|
|
|
|
Args:
|
|
aircraft_db_manager: An instance of AircraftDatabaseManager to perform DB operations.
|
|
main_window: A reference to the main application window (for GUI updates and messages).
|
|
"""
|
|
self.aircraft_db_manager = aircraft_db_manager
|
|
self.main_window = main_window
|
|
module_logger.debug("AircraftDBImporter initialized.")
|
|
|
|
def import_aircraft_database_with_progress(
|
|
self, csv_filepath: str, progress_dialog_ref: "ImportProgressDialog"
|
|
):
|
|
"""
|
|
Starts the aircraft database import process in a separate thread.
|
|
This method is intended to be called from the GUI thread.
|
|
|
|
Args:
|
|
csv_filepath: The path to the CSV file to import.
|
|
progress_dialog_ref: A reference to the ImportProgressDialog instance for updates.
|
|
"""
|
|
module_logger.info(
|
|
f"AircraftDBImporter: Requesting aircraft DB import with progress from: {csv_filepath}"
|
|
)
|
|
|
|
# Basic validation checks before starting the thread
|
|
if not self.aircraft_db_manager:
|
|
module_logger.error(
|
|
"AircraftDBImporter: AircraftDatabaseManager not initialized. Cannot import."
|
|
)
|
|
if progress_dialog_ref and progress_dialog_ref.winfo_exists():
|
|
# Schedule update to run on the GUI thread
|
|
self._schedule_gui_update(
|
|
progress_dialog_ref.import_finished,
|
|
False,
|
|
"Error: Database manager not active.",
|
|
)
|
|
elif self.main_window and hasattr(self.main_window, "show_error_message"):
|
|
self.main_window.show_error_message(
|
|
"Database Error", "Aircraft database manager not active."
|
|
)
|
|
return
|
|
|
|
if not (self.main_window and self.main_window.root.winfo_exists()):
|
|
module_logger.error(
|
|
"AircraftDBImporter: MainWindow not available to start import thread."
|
|
)
|
|
if progress_dialog_ref and progress_dialog_ref.winfo_exists():
|
|
self._schedule_gui_update(
|
|
progress_dialog_ref.import_finished,
|
|
False,
|
|
"Error: Main application window not available.",
|
|
)
|
|
return
|
|
|
|
# Notify the dialog that import has started
|
|
if progress_dialog_ref and progress_dialog_ref.winfo_exists():
|
|
self._schedule_gui_update(progress_dialog_ref.import_started)
|
|
|
|
# Start the import in a new daemon thread
|
|
import_thread = threading.Thread(
|
|
target=self._perform_db_import_threaded,
|
|
args=(csv_filepath, progress_dialog_ref),
|
|
daemon=True, # Daemon threads exit when the main program exits
|
|
name="AircraftDBImportWorker",
|
|
)
|
|
import_thread.start()
|
|
module_logger.info(
|
|
f"AircraftDBImporter: Import thread started for: {csv_filepath}"
|
|
)
|
|
|
|
def _perform_db_import_threaded(
|
|
self, csv_filepath: str, progress_dialog_ref: "ImportProgressDialog"
|
|
):
|
|
"""
|
|
Executes the database import process in a separate thread.
|
|
This method should NOT be called directly from the GUI thread.
|
|
"""
|
|
if not self.aircraft_db_manager:
|
|
module_logger.error(
|
|
"AircraftDBImporter: AircraftDBManager N/A in import thread."
|
|
)
|
|
if progress_dialog_ref and progress_dialog_ref.winfo_exists():
|
|
self._schedule_gui_update(
|
|
progress_dialog_ref.import_finished,
|
|
False,
|
|
"Internal Error: DB Manager missing.",
|
|
)
|
|
return
|
|
|
|
# Attempt to increase CSV field size limit to handle large fields
|
|
self._set_csv_field_size_limit()
|
|
|
|
module_logger.info(
|
|
f"AircraftDBImporter: Import thread: Starting row count for: {csv_filepath}"
|
|
)
|
|
|
|
# Count total rows for progress bar calculation
|
|
total_data_rows = self._count_csv_rows(csv_filepath)
|
|
if total_data_rows is None:
|
|
module_logger.error(
|
|
f"AircraftDBImporter: Could not count rows in CSV: {csv_filepath}"
|
|
)
|
|
if progress_dialog_ref and progress_dialog_ref.winfo_exists():
|
|
self._schedule_gui_update(
|
|
progress_dialog_ref.import_finished,
|
|
False,
|
|
f"Error: Could not read/count rows in '{os.path.basename(csv_filepath)}'. Check logs.",
|
|
)
|
|
return
|
|
if total_data_rows == 0:
|
|
module_logger.info(
|
|
f"AircraftDBImporter: CSV file '{csv_filepath}' is empty or header-only."
|
|
)
|
|
if progress_dialog_ref and progress_dialog_ref.winfo_exists():
|
|
self._schedule_gui_update(
|
|
progress_dialog_ref.update_progress,
|
|
0,
|
|
0,
|
|
0,
|
|
f"File '{os.path.basename(csv_filepath)}' is empty or header-only.",
|
|
)
|
|
self._schedule_gui_update(
|
|
progress_dialog_ref.import_finished,
|
|
True,
|
|
"Import complete: No data rows to import.",
|
|
)
|
|
return
|
|
|
|
# Initial progress update with total rows found
|
|
if progress_dialog_ref and progress_dialog_ref.winfo_exists():
|
|
self._schedule_gui_update(
|
|
progress_dialog_ref.update_progress,
|
|
0,
|
|
0,
|
|
total_data_rows,
|
|
f"Found {total_data_rows} data rows. Starting import from '{os.path.basename(csv_filepath)}'...",
|
|
)
|
|
|
|
# Define the progress callback function that will be passed to the DB manager
|
|
def import_progress_update_callback(
|
|
processed_csv_rows, imported_db_rows, total_for_cb
|
|
):
|
|
if progress_dialog_ref and progress_dialog_ref.winfo_exists():
|
|
# Schedule the GUI update on the main Tkinter thread
|
|
self._schedule_gui_update(
|
|
progress_dialog_ref.update_progress,
|
|
processed_csv_rows,
|
|
imported_db_rows,
|
|
total_for_cb,
|
|
f"Importing CSV row {processed_csv_rows}...",
|
|
)
|
|
|
|
# Perform the actual import via AircraftDatabaseManager
|
|
processed_final, imported_final = self.aircraft_db_manager.import_from_csv(
|
|
csv_filepath,
|
|
replace_existing=True, # Always replace existing for full import
|
|
progress_callback=import_progress_update_callback,
|
|
total_rows_for_callback=total_data_rows,
|
|
)
|
|
|
|
# Determine final message and success status
|
|
success = True
|
|
final_message = f"DB Import complete. Processed: {processed_final}, Imported/Updated: {imported_final}."
|
|
if imported_final == 0 and processed_final > 0 and total_data_rows > 0:
|
|
success = False # Consider it a failure if no rows were imported
|
|
final_message = f"DB Import: {processed_final} CSV data rows processed, 0 imported (check CSV format/logs)."
|
|
module_logger.error(f"AircraftDBImporter: {final_message}")
|
|
elif imported_final == 0 and processed_final == 0 and total_data_rows > 0:
|
|
success = False
|
|
final_message = f"DB Import failed. Could not process rows from '{os.path.basename(csv_filepath)}'."
|
|
module_logger.critical(f"AircraftDBImporter: {final_message}")
|
|
|
|
# Final update to the progress dialog
|
|
if progress_dialog_ref and progress_dialog_ref.winfo_exists():
|
|
self._schedule_gui_update(
|
|
progress_dialog_ref.update_progress,
|
|
processed_final,
|
|
imported_final,
|
|
total_data_rows,
|
|
"Finalizing import...",
|
|
)
|
|
self._schedule_gui_update(
|
|
progress_dialog_ref.import_finished, success, final_message
|
|
)
|
|
module_logger.info(
|
|
f"AircraftDBImporter: Import thread finished. Result: {final_message}"
|
|
)
|
|
|
|
def _count_csv_rows(self, csv_filepath: str) -> Optional[int]:
|
|
"""
|
|
Counts the number of data rows in a CSV file (excluding header).
|
|
|
|
Args:
|
|
csv_filepath: The path to the CSV file.
|
|
|
|
Returns:
|
|
The number of data rows, or None if the file cannot be read or is not found.
|
|
"""
|
|
# Temporarily increase CSV field size limit for robust row counting
|
|
self._set_csv_field_size_limit()
|
|
|
|
try:
|
|
with open(csv_filepath, "r", encoding="utf-8-sig") as f:
|
|
reader = csv.reader(f)
|
|
try:
|
|
next(reader) # Skip header row
|
|
except StopIteration:
|
|
return 0 # File is empty or only contains header
|
|
|
|
row_count = sum(1 for _ in reader)
|
|
module_logger.debug(
|
|
f"AircraftDBImporter: Counted {row_count} data rows in {csv_filepath}"
|
|
)
|
|
return row_count
|
|
except FileNotFoundError:
|
|
module_logger.error(
|
|
f"AircraftDBImporter: CSV file not found: {csv_filepath}"
|
|
)
|
|
return None
|
|
except csv.Error as e_csv:
|
|
module_logger.error(
|
|
f"AircraftDBImporter: CSV format error during row count of {csv_filepath}: {e_csv}",
|
|
exc_info=False,
|
|
)
|
|
return None
|
|
except Exception as e:
|
|
module_logger.error(
|
|
f"AircraftDBImporter: Unexpected error counting rows in {csv_filepath}: {e}",
|
|
exc_info=True,
|
|
)
|
|
return None
|
|
|
|
def _set_csv_field_size_limit(self):
|
|
"""
|
|
Attempts to increase the CSV field size limit to handle potentially large fields.
|
|
"""
|
|
try:
|
|
current_limit = csv.field_size_limit()
|
|
if CSV_FIELD_SIZE_LIMIT_BYTES > current_limit:
|
|
csv.field_size_limit(CSV_FIELD_SIZE_LIMIT_BYTES)
|
|
module_logger.debug(
|
|
f"AircraftDBImporter: Increased csv.field_size_limit from {current_limit} to {CSV_FIELD_SIZE_LIMIT_BYTES}."
|
|
)
|
|
except Exception as e:
|
|
module_logger.warning(
|
|
f"AircraftDBImporter: Error attempting to increase csv.field_size_limit: {e}. Proceeding with default limit.",
|
|
exc_info=False,
|
|
)
|
|
|
|
def _schedule_gui_update(self, callable_func: Callable, *args: Any):
|
|
"""
|
|
Schedules a callable function to be executed on the main Tkinter thread.
|
|
This is crucial for safely updating GUI elements from a worker thread.
|
|
|
|
Args:
|
|
callable_func: The function to call on the GUI thread.
|
|
*args: Arguments to pass to the callable function.
|
|
"""
|
|
if (
|
|
self.main_window
|
|
and hasattr(self.main_window, "root")
|
|
and self.main_window.root.winfo_exists()
|
|
):
|
|
try:
|
|
self.main_window.root.after(0, lambda: callable_func(*args))
|
|
except Exception as e:
|
|
module_logger.error(
|
|
f"AircraftDBImporter: Error scheduling GUI update from import thread: {e}",
|
|
exc_info=True,
|
|
)
|
|
else:
|
|
module_logger.warning(
|
|
f"AircraftDBImporter: Main window or root destroyed, cannot schedule GUI update for {callable_func.__name__}."
|
|
)
|