add save array per json

This commit is contained in:
VALLONGOL 2025-06-24 14:48:36 +02:00
parent 1527526188
commit 3f2ba715c1
3 changed files with 88 additions and 132 deletions

View File

@ -80,7 +80,7 @@
},
{
"column_name": "exp_pulse1_delay",
"data_path": "timer_data.blob.payload.exp_pulse1_delay",
"data_path": "timer_data.blob.payload.exp_pulse1_delay[0]",
"translate_with_enum": false
}
]

View File

@ -20,7 +20,7 @@ import ctypes
from ..utils.config_manager import ConfigManager
from ..core.file_reader import run_worker_process
from ..core.cpp_runner import run_cpp_converter
from ..core.data_structures import DataBatch
from ..core.data_structures import DataBatch, CtypesStructureBase
from ..core.data_enums import ENUM_REGISTRY, get_enum_name
from ..utils import logger
from ..gui.profile_editor_window import ProfileEditorWindow
@ -40,7 +40,6 @@ def _get_value_from_path(batch: DataBatch, field: ExportField) -> Any:
if path == "batch_id":
return batch.batch_id
# Split the path by dots and brackets to handle attributes and indices
parts = re.split(r"\.|\[", path)
current_obj = batch
@ -49,7 +48,6 @@ def _get_value_from_path(batch: DataBatch, field: ExportField) -> Any:
return "N/A"
if part.endswith("]"):
# This is an index access
index_str = part[:-1]
if not index_str.isdigit():
log.warning(f"Invalid index '{index_str}' in path: {path}")
@ -62,14 +60,11 @@ def _get_value_from_path(batch: DataBatch, field: ExportField) -> Any:
)
return "N/A"
else:
# This is an attribute access
current_obj = getattr(current_obj, part, None)
value = current_obj if current_obj is not None else "N/A"
# Handle translation for enums if the final value is an integer
if field.translate_with_enum and isinstance(value, int):
# For enum translation, we need the path without indices
enum_path = re.sub(r"\[\d+\]", "", path)
enum_class = ENUM_REGISTRY.get(enum_path)
if enum_class:
@ -81,6 +76,36 @@ def _get_value_from_path(batch: DataBatch, field: ExportField) -> Any:
return "N/A"
def _convert_ctypes_for_json(obj: Any) -> Any:
"""
Recursively converts ctypes objects into JSON-serializable Python types (lists, dicts, primitives).
"""
if isinstance(obj, (int, float, str, bool)) or obj is None:
return obj
# Handle basic ctypes values (c_int, c_float, etc.)
if isinstance(obj, (ctypes._SimpleCData)):
return obj.value
# Handle ctypes Structures
if isinstance(obj, CtypesStructureBase):
result = {}
for field_name, _ in obj._fields_:
# Skip fields that are not meant for direct data representation
if field_name.startswith("_"):
continue
value = getattr(obj, field_name)
result[field_name] = _convert_ctypes_for_json(value)
return result
# Handle ctypes Arrays
if isinstance(obj, ctypes.Array):
return [_convert_ctypes_for_json(item) for item in obj]
# Return the object itself if it's not a ctypes type we handle
return obj
class AppController:
"""The main controller of the application."""
@ -96,7 +121,6 @@ class AppController:
self.output_file_handles: Dict[str, Any] = {}
self.csv_writers: Dict[str, Any] = {}
# Buffer for JSON data
self.json_data_buffer: List[Dict[str, Any]] = []
self.last_generated_out_file: Optional[Path] = None
@ -110,18 +134,14 @@ class AppController:
if Path(last_file).is_file():
self.view.out_filepath_var.set(last_file)
self.on_out_config_changed()
if last_dir := self.config_manager.get("last_out_output_dir"):
self.view.out_output_dir_var.set(last_dir)
if last_file := self.config_manager.get("last_opened_rec_file"):
if Path(last_file).is_file():
self.view.rec_filepath_var.set(last_file)
self.on_rec_config_changed()
if last_dir := self.config_manager.get("last_rec_output_dir"):
self.view.rec_output_dir_var.set(last_dir)
profiles = self.config_manager.get_export_profiles()
self.view.update_export_profiles(
profiles=profiles,
@ -188,13 +208,12 @@ class AppController:
self.output_file_handles.clear()
self.csv_writers.clear()
self.active_export_profiles.clear()
self.json_data_buffer.clear() # Clear JSON buffer
self.json_data_buffer.clear()
try:
output_dir = Path(self.view.out_output_dir_var.get())
basename = self.view.out_basename_var.get()
profiles = self.config_manager.get_export_profiles()
use_full_path = self.view.out_use_full_path_var.get()
if self.view.out_output_csv_var.get():
profile = next(
(
@ -210,22 +229,18 @@ class AppController:
)
self.active_export_profiles["csv"] = profile
path = (output_dir / basename).with_suffix(".csv")
# Determine the delimiter based on the GUI checkbox
use_tab_delimiter = self.view.out_csv_use_tab_var.get()
delimiter = "\t" if use_tab_delimiter else ","
log.info(f"Preparing CSV file with '{delimiter}' as delimiter.")
fh = open(path, "w", encoding="utf-8", newline="")
self.output_file_handles["csv"] = fh
# Create the CSV writer with the chosen delimiter
csv_writer = csv.writer(fh, delimiter=delimiter)
self.csv_writers["csv"] = csv_writer
self.csv_writers["csv"].writerow(
[f.column_name for f in profile.fields]
)
headers = [
field.data_path if use_full_path else field.column_name
for field in profile.fields
]
self.csv_writers["csv"].writerow(headers)
if self.view.out_output_json_var.get():
profile = next(
(
@ -240,8 +255,6 @@ class AppController:
f"JSON profile '{self.view.out_json_profile_var.get()}' not found."
)
self.active_export_profiles["json"] = profile
# JSON file is no longer opened here, it's written at the end.
return True
except (IOError, ValueError) as e:
log.error(f"Failed to prepare output files: {e}")
@ -252,11 +265,9 @@ class AppController:
if self.is_processing:
log.warning("Processing already in progress.")
return
filepath_str = self.view.out_filepath_var.get()
if not all(
[
filepath_str,
self.view.out_filepath_var.get(),
self.view.out_output_dir_var.get(),
self.view.out_basename_var.get(),
]
@ -270,10 +281,9 @@ class AppController:
return
if not self._prepare_out_processor_files():
return
self.is_processing = True
self.view.start_processing_ui()
filepath_str = self.view.out_filepath_var.get()
self.config_manager.set("last_opened_out_file", filepath_str)
self.config_manager.set(
"last_out_output_dir", self.view.out_output_dir_var.get()
@ -282,7 +292,6 @@ class AppController:
"active_out_export_profile_name", self.view.out_csv_profile_var.get()
)
self.config_manager.save_config()
active_profile = self.active_export_profiles.get(
"csv"
) or self.active_export_profiles.get("json")
@ -301,7 +310,6 @@ class AppController:
raise ValueError(
"g_reconvert.exe path is not set or is invalid. Please set it in the Advanced Config."
)
rec_file = self.view.rec_filepath_var.get()
output_dir = self.view.rec_output_dir_var.get()
out_basename = self.view.rec_basename_var.get()
@ -309,17 +317,14 @@ class AppController:
raise ValueError(
"Missing required paths for C++ converter (REC file or Output)."
)
output_file_path = Path(output_dir) / f"{out_basename}.out"
self.last_generated_out_file = output_file_path
command = [
exe_path,
rec_file,
f"/o={str(output_file_path)}",
f"/n={self.view.rec_file_count_var.get()}",
]
if config.get("post_process"):
command.append(f"/p={config.get('post_process_level', '1')}")
if config.get("video_show"):
@ -330,7 +335,6 @@ class AppController:
command.append("/gps")
if config.get("silent_overwrite"):
command.append("//o")
log.info(f"Assembled C++ command: {' '.join(command)}")
return command
@ -345,7 +349,6 @@ class AppController:
log.error(f"Configuration error: {e}")
messagebox.showerror("Configuration Error", str(e), parent=self.view)
return
self.is_processing = True
self.view.start_processing_ui()
worker_args = (command_list, self.result_queue, output_dir)
@ -378,11 +381,8 @@ class AppController:
self.csv_writers.clear()
def handle_data_batch(self, batch: DataBatch):
"""Writes a data batch to CSV and buffers it for JSON."""
# Dato che il worker ora potrebbe restituire una struttura non completamente serializzabile
# tra processi (ctypes), è meglio gestire il batch come un dizionario o
# assicurarsi che la comunicazione avvenga in modo sicuro.
# Per ora, la logica principale non cambia, ma teniamo a mente questa potenziale complessità.
"""Writes a data batch to CSV and buffers it for JSON, converting ctypes."""
use_full_path = self.view.out_use_full_path_var.get()
if self.csv_writers.get("csv"):
profile = self.active_export_profiles["csv"]
@ -395,13 +395,11 @@ class AppController:
profile = self.active_export_profiles["json"]
row_dict = {}
for field in profile.fields:
value = _get_value_from_path(batch, field)
# I tipi ctypes non sono serializzabili in JSON, li convertiamo.
if isinstance(value, (ctypes.c_int, ctypes.c_uint, ctypes.c_float)):
value = value.value
elif isinstance(value, ctypes.Array):
value = list(value)
row_dict[field.column_name] = value
raw_value = _get_value_from_path(batch, field)
# Convert the entire value structure to be JSON-safe
serializable_value = _convert_ctypes_for_json(raw_value)
key = field.data_path if use_full_path else field.column_name
row_dict[key] = serializable_value
self.json_data_buffer.append(row_dict)
if batch.batch_id % 20 == 0:
@ -416,20 +414,17 @@ class AppController:
"JSON export was enabled, but no data batches were generated. Skipping file creation."
)
return
try:
output_dir = Path(self.view.out_output_dir_var.get())
basename = self.view.out_basename_var.get()
path = (output_dir / basename).with_suffix(".json")
log.info(
f"Writing {len(self.json_data_buffer)} records to JSON file: {path}"
)
with open(path, "w", encoding="utf-8") as f:
json.dump(self.json_data_buffer, f, indent=4)
log.info("JSON file written successfully.")
except (IOError, ValueError) as e:
except (IOError, TypeError) as e:
log.error(f"Failed to write JSON output file: {e}")
finally:
self.json_data_buffer.clear()
@ -455,16 +450,12 @@ class AppController:
def handle_worker_completion(self, msg: Dict[str, Any]):
status = "Interrupted" if msg.get("interrupted") else "Complete"
log.info(f"--- Process {status}. ---")
# Write buffered JSON data before closing files
if self.view.out_output_json_var.get():
self._write_json_buffer_to_file()
self._close_all_files()
self.is_processing = False
self.worker_process = None
self.view.update_ui_for_processing_state(False)
is_cpp_success = "Conversion process completed successfully" in msg.get(
"message", ""
)
@ -473,7 +464,6 @@ class AppController:
log.info(
f"C++ converter successfully generated: {self.last_generated_out_file}"
)
if stats := msg.get("stats"):
self._log_summary(stats)

View File

@ -11,30 +11,21 @@ import logging
from typing import Dict, Any, List
import queue
from .gui_utils import center_window
from ..utils import logger
from ..core.export_profiles import ExportProfile
log = logger.get_logger(__name__)
# --- Import Version Info FOR THE WRAPPER ITSELF ---
try:
# Use absolute import based on package name
from radar_data_reader import _version as wrapper_version
WRAPPER_APP_VERSION_STRING = f"{wrapper_version.__version__} ({wrapper_version.GIT_BRANCH}/{wrapper_version.GIT_COMMIT_HASH[:7]})"
WRAPPER_BUILD_INFO = f"Wrapper Built: {wrapper_version.BUILD_TIMESTAMP}"
except ImportError:
# This might happen if you run the wrapper directly from source
# without generating its _version.py first (if you use that approach for the wrapper itself)
WRAPPER_APP_VERSION_STRING = "(Dev Wrapper)"
WRAPPER_BUILD_INFO = "Wrapper build time unknown"
# --- Constants for Version Generation ---
DEFAULT_VERSION = "0.0.0+unknown"
DEFAULT_COMMIT = "Unknown"
DEFAULT_BRANCH = "Unknown"
# --- End Constants ---
class MainWindow(tk.Frame):
"""The main application window (View)."""
@ -70,10 +61,9 @@ class MainWindow(tk.Frame):
self.out_output_dir_var = tk.StringVar()
self.out_basename_var = tk.StringVar()
self.out_output_csv_var = tk.BooleanVar(value=True)
self.out_csv_use_tab_var = tk.BooleanVar(
value=False
) # New variable for tab separator
self.out_csv_use_tab_var = tk.BooleanVar(value=False)
self.out_output_json_var = tk.BooleanVar(value=False)
self.out_use_full_path_var = tk.BooleanVar(value=False) # New variable
self.out_csv_profile_var = tk.StringVar()
self.out_json_profile_var = tk.StringVar()
@ -145,7 +135,7 @@ class MainWindow(tk.Frame):
self.out_browse_button.grid(row=0, column=2, padx=5, pady=5)
self.out_filepath_var.trace_add("write", self.controller.on_out_config_changed)
output_frame = ttk.LabelFrame(parent, text="CSV/JSON Output Configuration")
output_frame = ttk.LabelFrame(parent, text="Output Configuration")
output_frame.grid(row=1, column=0, columnspan=3, sticky="ew", padx=5, pady=5)
output_frame.columnconfigure(1, weight=1)
ttk.Label(output_frame, text="Output Directory:").grid(
@ -154,7 +144,6 @@ class MainWindow(tk.Frame):
out_dir_entry = ttk.Entry(output_frame, textvariable=self.out_output_dir_var)
out_dir_entry.grid(row=0, column=1, sticky="ew", padx=5)
# --- Buttons Frame ---
out_dir_buttons_frame = ttk.Frame(output_frame)
out_dir_buttons_frame.grid(row=0, column=2, padx=5)
ttk.Button(
@ -177,49 +166,49 @@ class MainWindow(tk.Frame):
row=1, column=1, columnspan=2, sticky="ew", padx=5
)
formats_frame = ttk.LabelFrame(parent, text="Output Formats & Profiles")
# --- Reorganized Options Grid ---
formats_frame = ttk.LabelFrame(parent, text="Output Formats & Options")
formats_frame.grid(row=2, column=0, columnspan=3, sticky="ew", padx=5, pady=5)
formats_frame.columnconfigure(1, weight=1)
formats_frame.columnconfigure(2, weight=1) # Add weight to third column
# CSV Options
csv_options_frame = ttk.Frame(formats_frame)
csv_options_frame.grid(row=0, column=0, columnspan=3, sticky="ew")
ttk.Checkbutton(
csv_options_frame,
text="Generate .csv file",
variable=self.out_output_csv_var,
).pack(side=tk.LEFT, padx=(5, 10))
formats_frame, text="Generate .csv file", variable=self.out_output_csv_var
).grid(row=0, column=0, sticky="w", padx=5, pady=2)
self.out_csv_profile_combobox = ttk.Combobox(
csv_options_frame,
formats_frame,
textvariable=self.out_csv_profile_var,
state="readonly",
width=20,
width=25,
)
self.out_csv_profile_combobox.pack(side=tk.LEFT, padx=5)
ttk.Checkbutton(
csv_options_frame,
text="Use Tab Separator",
variable=self.out_csv_use_tab_var,
).pack(side=tk.LEFT, padx=(10, 5))
self.out_csv_profile_combobox.grid(row=0, column=1, sticky="w", padx=5)
# JSON Options
json_options_frame = ttk.Frame(formats_frame)
json_options_frame.grid(row=1, column=0, columnspan=3, sticky="ew")
ttk.Checkbutton(
json_options_frame,
text="Generate .json file",
variable=self.out_output_json_var,
).pack(side=tk.LEFT, padx=(5, 10))
formats_frame, text="Generate .json file", variable=self.out_output_json_var
).grid(row=1, column=0, sticky="w", padx=5, pady=2)
self.out_json_profile_combobox = ttk.Combobox(
json_options_frame,
formats_frame,
textvariable=self.out_json_profile_var,
state="readonly",
width=20,
width=25,
)
self.out_json_profile_combobox.pack(side=tk.LEFT, padx=5)
self.out_json_profile_combobox.grid(row=1, column=1, sticky="w", padx=5)
# General Formatting Options in a separate sub-frame for alignment
options_subframe = ttk.Frame(formats_frame)
options_subframe.grid(row=0, column=2, rowspan=2, sticky="w", padx=(20, 5))
ttk.Checkbutton(
options_subframe,
text="Use Tab Separator (CSV)",
variable=self.out_csv_use_tab_var,
).pack(anchor="w")
ttk.Checkbutton(
options_subframe,
text="Use Full Path for Headers",
variable=self.out_use_full_path_var,
).pack(anchor="w")
action_frame = ttk.Frame(parent)
action_frame.grid(row=3, column=0, columnspan=3, pady=(10, 0))
@ -243,11 +232,9 @@ class MainWindow(tk.Frame):
def _create_rec_converter_tab(self, parent):
parent.columnconfigure(1, weight=1)
input_frame = ttk.LabelFrame(parent, text="Input REC Sequence")
input_frame.grid(row=0, column=0, columnspan=3, sticky="ew", padx=5, pady=5)
input_frame.columnconfigure(1, weight=1)
ttk.Label(input_frame, text="First .rec File:").grid(
row=0, column=0, padx=5, pady=5, sticky="w"
)
@ -258,19 +245,16 @@ class MainWindow(tk.Frame):
ttk.Button(
input_frame, text="Browse...", command=self.controller.select_rec_file
).grid(row=0, column=2, padx=5)
ttk.Label(input_frame, text="Number of Files (/n):").grid(
row=1, column=0, padx=5, pady=5, sticky="w"
)
rec_file_count_spinbox = ttk.Spinbox(
ttk.Spinbox(
input_frame,
from_=1,
to=1000,
textvariable=self.rec_file_count_var,
width=10,
)
rec_file_count_spinbox.grid(row=1, column=1, padx=5, pady=5, sticky="w")
).grid(row=1, column=1, padx=5, pady=5, sticky="w")
self.rec_filepath_var.trace_add("write", self.controller.on_rec_config_changed)
self.rec_file_count_var.trace_add(
"write", self.controller.on_rec_config_changed
@ -279,13 +263,11 @@ class MainWindow(tk.Frame):
output_frame = ttk.LabelFrame(parent, text="Generated .out File")
output_frame.grid(row=1, column=0, columnspan=3, sticky="ew", padx=5, pady=5)
output_frame.columnconfigure(1, weight=1)
ttk.Label(output_frame, text="Output Directory:").grid(
row=0, column=0, padx=5, pady=5, sticky="w"
)
rec_dir_entry = ttk.Entry(output_frame, textvariable=self.rec_output_dir_var)
rec_dir_entry.grid(row=0, column=1, sticky="ew", padx=5)
rec_dir_buttons_frame = ttk.Frame(output_frame)
rec_dir_buttons_frame.grid(row=0, column=2, padx=5)
ttk.Button(
@ -300,7 +282,6 @@ class MainWindow(tk.Frame):
self.rec_output_dir_var.get()
),
).pack(side=tk.LEFT, padx=(5, 0))
ttk.Label(output_frame, text="Generated Filename:").grid(
row=1, column=0, padx=5, pady=5, sticky="w"
)
@ -313,21 +294,18 @@ class MainWindow(tk.Frame):
action_frame = ttk.Frame(parent)
action_frame.grid(row=2, column=0, columnspan=3, pady=(10, 0))
self.rec_convert_button = ttk.Button(
action_frame,
text="Convert REC to OUT",
command=self.controller.start_rec_conversion,
)
self.rec_convert_button.pack(side=tk.LEFT, padx=5)
self.rec_config_button = ttk.Button(
action_frame,
text="g_reconverter Advanced Config...",
command=self.controller.open_rec_config_editor,
)
self.rec_config_button.pack(side=tk.LEFT, padx=5)
self.process_generated_out_button = ttk.Button(
action_frame,
text="Process Generated .out File ->",
@ -341,28 +319,24 @@ class MainWindow(tk.Frame):
parent, text="Live Data & Progress (.out Processor)"
)
status_frame.columnconfigure(1, weight=1)
self.progress_bar = ttk.Progressbar(
status_frame, variable=self.progress_bar_var, maximum=100
)
self.progress_bar.grid(
row=0, column=0, columnspan=2, sticky="ew", padx=5, pady=(5, 2)
)
ttk.Label(status_frame, text="Progress:", anchor="e").grid(
row=1, column=0, padx=(10, 5), pady=2, sticky="e"
)
ttk.Label(status_frame, textvariable=self.progress_text_var, anchor="w").grid(
row=1, column=1, padx=5, pady=2, sticky="w"
)
ttk.Label(status_frame, text="Batches Found:", anchor="e").grid(
row=2, column=0, padx=(10, 5), pady=2, sticky="e"
)
ttk.Label(status_frame, textvariable=self.batches_found_var, anchor="w").grid(
row=2, column=1, padx=5, pady=2, sticky="w"
)
return status_frame
def _create_log_console_frame(self, parent):
@ -374,7 +348,6 @@ class MainWindow(tk.Frame):
log_frame, state=tk.DISABLED, wrap=tk.WORD, bg="#f0f0f0"
)
self.log_widget.grid(row=0, column=0, sticky="nsew", padx=5, pady=5)
self.log_widget.tag_config("INFO", foreground="black")
self.log_widget.tag_config("ERROR", foreground="red", font=("", 0, "bold"))
self.log_widget.tag_config("SUCCESS", foreground="green")
@ -388,7 +361,6 @@ class MainWindow(tk.Frame):
def update_export_profiles(self, profiles: List[ExportProfile], **kwargs):
profile_names = [p.name for p in profiles] if profiles else []
active_out_profile = kwargs.get("active_out_profile", "")
for combo, var, active_name in [
(
self.out_csv_profile_combobox,
@ -419,17 +391,13 @@ class MainWindow(tk.Frame):
def update_ui_for_processing_state(self, is_processing: bool):
state = tk.DISABLED if is_processing else tk.NORMAL
self.out_browse_button.config(state=state)
self.out_process_button.config(state=state)
self.rec_convert_button.config(state=state)
self.rec_config_button.config(state=state)
if is_processing:
self.process_generated_out_button.config(state=tk.DISABLED)
self.out_stop_button.config(state=tk.NORMAL if is_processing else tk.DISABLED)
if is_processing:
self.status_bar_var.set("Processing... Please wait.")
self.master.config(cursor="watch")
@ -440,11 +408,9 @@ class MainWindow(tk.Frame):
self.master.config(cursor="")
def update_rec_tab_buttons_state(self, conversion_successful: bool):
"""Called by the controller to update button states after conversion."""
if conversion_successful:
self.process_generated_out_button.config(state=tk.NORMAL)
else:
self.process_generated_out_button.config(state=tk.DISABLED)
self.process_generated_out_button.config(
state=tk.NORMAL if conversion_successful else tk.DISABLED
)
def poll_result_queue(self):
try:
@ -467,10 +433,10 @@ class MainWindow(tk.Frame):
elif msg_type == "progress":
blocks_done = msg.get("blocks_done", 0)
self.batches_found_var.set(str(msg.get("batch_id", "N/A")))
self.progress_text_var.set(
f"Block {blocks_done} / {self.total_blocks_for_progress}"
)
if self.total_blocks_for_progress > 0:
self.progress_text_var.set(
f"Block {blocks_done} / {self.total_blocks_for_progress}"
)
self.progress_bar_var.set(
(blocks_done / self.total_blocks_for_progress) * 100
)