From aae47125cd67db8f9018199c09de394017a11ce4 Mon Sep 17 00:00:00 2001 From: NguyenND Date: Fri, 22 May 2026 09:20:13 -0400 Subject: [PATCH] feat: add SQL dump import support to ImportDialog Extends ImportDialog to handle .sql dump files alongside CSV/JSON. Adds _SqlImportWorker that splits SQL with sqlparse (semicolon fallback), executes each statement via driver.execute_query(), and collects non-fatal errors. UI shows a text preview of the dump, statement count, a stop-on-error checkbox, and a final summary with any per-statement errors. Co-Authored-By: Claude Sonnet 4.6 --- app/ui/import_dialog.py | 374 +++++++++++++++++++++++++++++++--------- 1 file changed, 294 insertions(+), 80 deletions(-) diff --git a/app/ui/import_dialog.py b/app/ui/import_dialog.py index cf9282d..ec2f220 100644 --- a/app/ui/import_dialog.py +++ b/app/ui/import_dialog.py @@ -1,10 +1,13 @@ """ -Import CSV / JSON into a database table. +Import CSV / JSON / SQL dump into a database. -Flow: - 1. User picks a file (.csv or .json) - 2. A preview of the first N rows is shown - 3. User confirms → rows are inserted via the driver's insert_row() +CSV / JSON flow: + 1. Pick file → parse → table preview → insert rows via driver.insert_row() + +SQL dump flow: + 1. Pick .sql file → split into statements (sqlparse) → text preview + 2. Execute each statement via driver.execute_query() + 3. Report progress; optional "stop on first error" mode """ import csv import json @@ -14,18 +17,21 @@ from PyQt6.QtWidgets import ( QLabel, QLineEdit, QPushButton, QComboBox, QTableWidget, QTableWidgetItem, QHeaderView, QDialogButtonBox, QFileDialog, QProgressBar, - QMessageBox, QCheckBox, + QMessageBox, QCheckBox, QStackedWidget, QPlainTextEdit, + QWidget, ) from PyQt6.QtCore import Qt, QThread, pyqtSignal +from PyQt6.QtGui import QFont -_PREVIEW_ROWS = 50 +_PREVIEW_ROWS = 50 +_PREVIEW_SQL_LINES = 200 # lines of SQL text shown in preview -# ── Background import worker ────────────────────────────────────────────────── +# ── CSV / JSON row-insert worker ────────────────────────────────────────────── -class _ImportWorker(QThread): - progress = pyqtSignal(int) # rows inserted so far - finished = pyqtSignal(int) # total rows inserted +class _RowImportWorker(QThread): + progress = pyqtSignal(int) # rows inserted so far + finished = pyqtSignal(int) # total inserted error = pyqtSignal(str) def __init__(self, driver, database: str, table: str, @@ -34,7 +40,7 @@ class _ImportWorker(QThread): self._driver = driver self._database = database self._table = table - self._rows = rows # list of dicts {col: value} + self._rows = rows def run(self): inserted = 0 @@ -49,34 +55,89 @@ class _ImportWorker(QThread): self.error.emit(f"Row {inserted + 1}: {e}") +# ── SQL dump execution worker ───────────────────────────────────────────────── + +class _SqlImportWorker(QThread): + progress = pyqtSignal(int) # statements executed so far + stmt_error = pyqtSignal(int, str) # (stmt_index, message) — non-fatal + finished = pyqtSignal(int, int) # (executed, total) + fatal = pyqtSignal(str) # stopped on error + + def __init__(self, driver, database: str, sql_text: str, + stop_on_error: bool = False, parent=None): + super().__init__(parent) + self._driver = driver + self._database = database + self._sql_text = sql_text + self._stop_on_error = stop_on_error + + def run(self): + try: + import sqlparse + stmts = [s.strip() for s in sqlparse.split(self._sql_text) if s.strip()] + except ImportError: + # Fallback: naive semicolon split + stmts = [s.strip() for s in self._sql_text.split(";") if s.strip()] + + total = len(stmts) + executed = 0 + for i, stmt in enumerate(stmts): + try: + self._driver.execute_query(stmt) + executed += 1 + except Exception as e: + self.stmt_error.emit(i, str(e)) + if self._stop_on_error: + self.fatal.emit( + f"Stopped at statement {i + 1}/{total}:\n{e}" + ) + return + if (i + 1) % 20 == 0: + self.progress.emit(i + 1) + + self.progress.emit(total) + self.finished.emit(executed, total) + + # ── Dialog ──────────────────────────────────────────────────────────────────── class ImportDialog(QDialog): - """Select a CSV or JSON file and import its contents into a table.""" + """Import CSV, JSON, or SQL dump into the database.""" - def __init__(self, driver, database: str, table: str, parent=None): + # Internal mode constants + _MODE_TABLE = 0 # CSV / JSON + _MODE_SQL = 1 # SQL dump + + def __init__(self, driver, database: str, table: str = "", parent=None): super().__init__(parent) self._driver = driver self._database = database self._table = table - self._rows: list = [] # parsed rows ready for import - self._worker = None + self._rows: list = [] # parsed rows (CSV/JSON mode) + self._sql_text = "" # raw SQL text (SQL mode) + self._sql_stmts = 0 # statement count (SQL mode) + self._worker = None + self._mode = self._MODE_TABLE + self._errors: list[str] = [] # non-fatal SQL errors collected - self.setWindowTitle(f"Import into {table}") + self.setWindowTitle("Import Data") self.setModal(True) - self.setMinimumSize(640, 480) + self.setMinimumSize(680, 520) self._build_ui() # ── UI ──────────────────────────────────────────────────────────────────── def _build_ui(self): root = QVBoxLayout(self) + root.setSpacing(8) - # File picker row + # ── File picker ─────────────────────────────────────────────────────── file_row = QHBoxLayout() self._path_edit = QLineEdit() self._path_edit.setReadOnly(True) - self._path_edit.setPlaceholderText("Select a .csv or .json file…") + self._path_edit.setPlaceholderText( + "Select a .csv, .json, or .sql file…" + ) browse_btn = QPushButton("Browse…") browse_btn.setFixedWidth(80) browse_btn.clicked.connect(self._browse) @@ -84,38 +145,56 @@ class ImportDialog(QDialog): file_row.addWidget(browse_btn) root.addLayout(file_row) - # CSV options (hidden until a CSV is selected) - self._csv_opts = QHBoxLayout() + # ── CSV-only options ────────────────────────────────────────────────── self._csv_opts_widget = self._build_csv_opts() - root.addWidget(self._csv_opts_widget) self._csv_opts_widget.setVisible(False) + root.addWidget(self._csv_opts_widget) - # Skip-header checkbox self._header_cb = QCheckBox("First row is a header (CSV only)") self._header_cb.setChecked(True) self._header_cb.toggled.connect(self._reload_preview) + self._header_cb.setVisible(False) root.addWidget(self._header_cb) - # Preview table - root.addWidget(QLabel("Preview (first 50 rows):")) - self._preview = QTableWidget(0, 0) - self._preview.setAlternatingRowColors(True) - self._preview.setEditTriggers(QTableWidget.EditTrigger.NoEditTriggers) - self._preview.horizontalHeader().setSectionResizeMode( - QHeaderView.ResizeMode.ResizeToContents - ) - self._preview.verticalHeader().setDefaultSectionSize(22) - self._preview.setSelectionMode(QTableWidget.SelectionMode.NoSelection) - root.addWidget(self._preview, 1) + # ── SQL-only options ────────────────────────────────────────────────── + self._sql_opts_widget = self._build_sql_opts() + self._sql_opts_widget.setVisible(False) + root.addWidget(self._sql_opts_widget) - # Status / progress + # ── Preview stack: page 0 = table, page 1 = SQL text ───────────────── + self._preview_label = QLabel("Preview:") + root.addWidget(self._preview_label) + + self._preview_stack = QStackedWidget() + + self._preview_table = QTableWidget(0, 0) + self._preview_table.setAlternatingRowColors(True) + self._preview_table.setEditTriggers( + QTableWidget.EditTrigger.NoEditTriggers) + self._preview_table.horizontalHeader().setSectionResizeMode( + QHeaderView.ResizeMode.ResizeToContents) + self._preview_table.verticalHeader().setDefaultSectionSize(22) + self._preview_table.setSelectionMode( + QTableWidget.SelectionMode.NoSelection) + self._preview_stack.addWidget(self._preview_table) # idx 0 + + self._preview_sql = QPlainTextEdit() + self._preview_sql.setReadOnly(True) + self._preview_sql.setFont(QFont("Consolas", 10)) + self._preview_sql.setLineWrapMode(QPlainTextEdit.LineWrapMode.NoWrap) + self._preview_stack.addWidget(self._preview_sql) # idx 1 + + root.addWidget(self._preview_stack, 1) + + # ── Status / progress ───────────────────────────────────────────────── self._status_lbl = QLabel("") root.addWidget(self._status_lbl) + self._progress = QProgressBar() self._progress.setVisible(False) root.addWidget(self._progress) - # Buttons + # ── Buttons ─────────────────────────────────────────────────────────── self._btns = QDialogButtonBox( QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel @@ -127,8 +206,7 @@ class ImportDialog(QDialog): self._btns.rejected.connect(self.reject) root.addWidget(self._btns) - def _build_csv_opts(self) -> QHBoxLayout: - from PyQt6.QtWidgets import QWidget + def _build_csv_opts(self) -> QWidget: w = QWidget() lay = QHBoxLayout(w) lay.setContentsMargins(0, 0, 0, 0) @@ -142,69 +220,116 @@ class ImportDialog(QDialog): lay.addStretch() return w + def _build_sql_opts(self) -> QWidget: + w = QWidget() + lay = QHBoxLayout(w) + lay.setContentsMargins(0, 0, 0, 0) + self._stop_on_error_cb = QCheckBox("Stop on first error") + self._stop_on_error_cb.setChecked(False) + self._stop_on_error_cb.setToolTip( + "When unchecked, errors are logged and execution continues.\n" + "When checked, the import halts at the first failing statement." + ) + lay.addWidget(self._stop_on_error_cb) + lay.addStretch() + return w + # ── File loading ────────────────────────────────────────────────────────── def _browse(self): path, _ = QFileDialog.getOpenFileName( self, "Open file", "", - "CSV / JSON files (*.csv *.json);;All files (*)" + "Supported files (*.csv *.json *.sql);;" + "CSV files (*.csv);;" + "JSON files (*.json);;" + "SQL dump files (*.sql);;" + "All files (*)" ) if not path: return self._path_edit.setText(path) ext = os.path.splitext(path)[1].lower() - self._csv_opts_widget.setVisible(ext == ".csv") - self._header_cb.setVisible(ext == ".csv") + is_csv = ext == ".csv" + is_sql = ext == ".sql" + + self._csv_opts_widget.setVisible(is_csv) + self._header_cb.setVisible(is_csv) + self._sql_opts_widget.setVisible(is_sql) + + self._mode = self._MODE_SQL if is_sql else self._MODE_TABLE self._reload_preview() def _delimiter(self) -> str: - mapping = {0: ",", 1: ";", 2: "\t", 3: "|"} - return mapping.get(self._delim_combo.currentIndex(), ",") + return {0: ",", 1: ";", 2: "\t", 3: "|"}.get( + self._delim_combo.currentIndex(), "," + ) def _reload_preview(self): path = self._path_edit.text() if not path: return ext = os.path.splitext(path)[1].lower() + + if ext == ".sql": + self._load_sql_preview(path) + elif ext == ".csv": + self._load_csv_preview(path) + elif ext == ".json": + self._load_json_preview(path) + else: + self._status_lbl.setText("Unsupported file type.") + + # ── CSV / JSON loading ──────────────────────────────────────────────────── + + def _load_csv_preview(self, path: str): try: - if ext == ".csv": - self._rows = self._parse_csv(path) - elif ext == ".json": - self._rows = self._parse_json(path) - else: - self._status_lbl.setText("Unsupported file type.") - return + self._rows = self._parse_csv(path) except Exception as e: self._status_lbl.setText(f"Parse error: {e}") self._rows = [] self._ok_btn.setEnabled(False) return - - self._populate_preview(self._rows[:_PREVIEW_ROWS]) + self._preview_stack.setCurrentIndex(0) + self._preview_label.setText(f"Preview (first {_PREVIEW_ROWS} rows):") + self._populate_table_preview(self._rows[:_PREVIEW_ROWS]) + target = f"'{self._database}'.'{self._table}'" if self._table else f"'{self._database}'" self._status_lbl.setText( - f"{len(self._rows)} row(s) ready to import into " - f"'{self._database}'.'{self._table}'" + f"{len(self._rows):,} row(s) ready to import into {target}" + ) + self._ok_btn.setEnabled(bool(self._rows)) + + def _load_json_preview(self, path: str): + try: + self._rows = self._parse_json(path) + except Exception as e: + self._status_lbl.setText(f"Parse error: {e}") + self._rows = [] + self._ok_btn.setEnabled(False) + return + self._preview_stack.setCurrentIndex(0) + self._preview_label.setText(f"Preview (first {_PREVIEW_ROWS} rows):") + self._populate_table_preview(self._rows[:_PREVIEW_ROWS]) + target = f"'{self._database}'.'{self._table}'" if self._table else f"'{self._database}'" + self._status_lbl.setText( + f"{len(self._rows):,} row(s) ready to import into {target}" ) self._ok_btn.setEnabled(bool(self._rows)) def _parse_csv(self, path: str) -> list: - rows = [] with open(path, newline="", encoding="utf-8-sig") as f: reader = csv.reader(f, delimiter=self._delimiter()) all_rows = list(reader) if not all_rows: return [] if self._header_cb.isChecked(): - headers = all_rows[0] - data_rows = all_rows[1:] + headers, data_rows = all_rows[0], all_rows[1:] else: headers = [f"col{i+1}" for i in range(len(all_rows[0]))] data_rows = all_rows - for row in data_rows: - # Pad short rows, truncate long ones - padded = (row + [""] * len(headers))[: len(headers)] - rows.append(dict(zip(headers, padded))) - return rows + return [ + dict(zip(headers, (row + [""] * len(headers))[: len(headers)])) + for row in data_rows + ] def _parse_json(self, path: str) -> list: with open(path, encoding="utf-8") as f: @@ -212,57 +337,146 @@ class ImportDialog(QDialog): if isinstance(data, list): return [r for r in data if isinstance(r, dict)] if isinstance(data, dict): - # Support {rows: [...]} or {data: [...]} wrappers for key in ("rows", "data", "records", "items"): if isinstance(data.get(key), list): return data[key] raise ValueError("JSON must be an array of objects or {rows: [...]}") - # ── Preview ─────────────────────────────────────────────────────────────── + # ── SQL loading ─────────────────────────────────────────────────────────── - def _populate_preview(self, rows: list): + def _load_sql_preview(self, path: str): + try: + with open(path, encoding="utf-8", errors="replace") as f: + self._sql_text = f.read() + except Exception as e: + self._status_lbl.setText(f"Read error: {e}") + self._sql_text = "" + self._sql_stmts = 0 + self._ok_btn.setEnabled(False) + return + + # Count statements + try: + import sqlparse + stmts = [s for s in sqlparse.split(self._sql_text) if s.strip()] + except ImportError: + stmts = [s for s in self._sql_text.split(";") if s.strip()] + self._sql_stmts = len(stmts) + + # Show first N lines in text preview + lines = self._sql_text.splitlines() + preview_text = "\n".join(lines[:_PREVIEW_SQL_LINES]) + if len(lines) > _PREVIEW_SQL_LINES: + preview_text += f"\n\n… ({len(lines) - _PREVIEW_SQL_LINES:,} more lines)" + self._preview_sql.setPlainText(preview_text) + + self._preview_stack.setCurrentIndex(1) + self._preview_label.setText( + f"Preview (first {_PREVIEW_SQL_LINES} lines):" + ) + self._status_lbl.setText( + f"{self._sql_stmts:,} statement(s) ready to execute " + f"on database '{self._database}'" + ) + self._ok_btn.setEnabled(self._sql_stmts > 0) + + # ── Table preview (CSV/JSON) ─────────────────────────────────────────────── + + def _populate_table_preview(self, rows: list): if not rows: - self._preview.setRowCount(0) - self._preview.setColumnCount(0) + self._preview_table.setRowCount(0) + self._preview_table.setColumnCount(0) return headers = list(rows[0].keys()) - self._preview.setColumnCount(len(headers)) - self._preview.setHorizontalHeaderLabels(headers) - self._preview.setRowCount(len(rows)) + self._preview_table.setColumnCount(len(headers)) + self._preview_table.setHorizontalHeaderLabels(headers) + self._preview_table.setRowCount(len(rows)) for r, row in enumerate(rows): for c, key in enumerate(headers): val = row.get(key, "") - self._preview.setItem( + self._preview_table.setItem( r, c, QTableWidgetItem("" if val is None else str(val)) ) - # ── Import ──────────────────────────────────────────────────────────────── + # ── Import dispatch ─────────────────────────────────────────────────────── def _start_import(self): + self._ok_btn.setEnabled(False) + self._errors.clear() + + if self._mode == self._MODE_SQL: + self._start_sql_import() + else: + self._start_row_import() + + def _start_row_import(self): if not self._rows: return - self._ok_btn.setEnabled(False) self._progress.setMaximum(len(self._rows)) self._progress.setValue(0) self._progress.setVisible(True) - self._worker = _ImportWorker( + self._worker = _RowImportWorker( self._driver, self._database, self._table, self._rows, parent=self ) self._worker.progress.connect(self._progress.setValue) - self._worker.finished.connect(self._on_done) - self._worker.error.connect(self._on_error) + self._worker.finished.connect(self._on_row_done) + self._worker.error.connect(self._on_fatal_error) self._worker.start() - def _on_done(self, count: int): + def _start_sql_import(self): + if not self._sql_text.strip(): + return + self._progress.setMaximum(self._sql_stmts) + self._progress.setValue(0) + self._progress.setVisible(True) + self._status_lbl.setText("Executing SQL statements…") + + self._worker = _SqlImportWorker( + self._driver, self._database, self._sql_text, + stop_on_error=self._stop_on_error_cb.isChecked(), + parent=self, + ) + self._worker.progress.connect(self._progress.setValue) + self._worker.stmt_error.connect(self._on_stmt_error) + self._worker.finished.connect(self._on_sql_done) + self._worker.fatal.connect(self._on_fatal_error) + self._worker.start() + + # ── Worker callbacks ────────────────────────────────────────────────────── + + def _on_row_done(self, count: int): self._progress.setValue(count) QMessageBox.information( self, "Import Complete", - f"Successfully imported {count} row(s) into '{self._table}'." + f"Successfully imported {count:,} row(s) into '{self._table}'." ) self.accept() - def _on_error(self, msg: str): + def _on_sql_done(self, executed: int, total: int): + self._progress.setValue(total) + skipped = total - executed + msg = ( + f"Executed {executed:,} of {total:,} statement(s) " + f"on database '{self._database}'." + ) + if self._errors: + msg += f"\n\n⚠️ {len(self._errors)} statement(s) produced errors:" + msg += "\n" + "\n".join(self._errors[:10]) + if len(self._errors) > 10: + msg += f"\n… and {len(self._errors) - 10} more" + QMessageBox.warning(self, "Import Finished with Errors", msg) + else: + QMessageBox.information(self, "Import Complete", msg) + self.accept() + + def _on_stmt_error(self, idx: int, msg: str): + self._errors.append(f"Statement {idx + 1}: {msg}") + self._status_lbl.setText( + f"Running… ({len(self._errors)} error(s) so far)" + ) + + def _on_fatal_error(self, msg: str): self._progress.setVisible(False) self._ok_btn.setEnabled(True) QMessageBox.critical(self, "Import Error", f"Import failed:\n{msg}")