From 02f27dc294c53a78a8210b06835e80729c735476 Mon Sep 17 00:00:00 2001 From: Abeautifulsnow Date: Mon, 28 Sep 2026 21:38:32 +0800 Subject: [PATCH] feat(table-import): add DuckDB Parquet imports --- agents/drivers/duckdb/src/runtime.rs | 126 +++++- agents/drivers/duckdb/src/wire.rs | 3 + .../duckdb/tests/duckdb_worker_process.rs | 13 +- .../components/import/TableImportDialog.vue | 45 ++- .../TableImportDialog.batchSelection.spec.ts | 31 +- apps/desktop/src/i18n/locales/az.ts | 3 + apps/desktop/src/i18n/locales/en.ts | 3 + apps/desktop/src/i18n/locales/es.ts | 3 + apps/desktop/src/i18n/locales/it.ts | 3 + apps/desktop/src/i18n/locales/ja.ts | 3 + apps/desktop/src/i18n/locales/ko.ts | 3 + apps/desktop/src/i18n/locales/pt-BR.ts | 3 + apps/desktop/src/i18n/locales/ru.ts | 3 + apps/desktop/src/i18n/locales/tr.ts | 3 + apps/desktop/src/i18n/locales/zh-CN.ts | 3 + apps/desktop/src/i18n/locales/zh-TW.ts | 3 + apps/desktop/src/lib/backend/http.ts | 4 + apps/desktop/src/lib/backend/tauri.ts | 4 +- .../dbx-core/examples/table_import_bench.rs | 2 + crates/dbx-core/src/data/table_import.rs | 370 +++++++++++++++++- .../src/db/duckdb_worker_process.rs | 31 +- .../src/db/duckdb_worker_protocol.rs | 7 + crates/dbx-web/src/routes/table_import.rs | 46 ++- docs/content/docs/table-import.cn.mdx | 5 +- docs/content/docs/table-import.mdx | 5 +- src-tauri/src/commands/table_import.rs | 7 +- 26 files changed, 686 insertions(+), 46 deletions(-) diff --git a/agents/drivers/duckdb/src/runtime.rs b/agents/drivers/duckdb/src/runtime.rs index 9d70df898a..e8dde087f3 100644 --- a/agents/drivers/duckdb/src/runtime.rs +++ b/agents/drivers/duckdb/src/runtime.rs @@ -50,13 +50,17 @@ impl DuckDbWorkerSession { pub fn execute(&mut self, params: DuckDbWorkerExecuteParams) -> Result { let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone(); let locked = connection.lock().map_err(|e| e.to_string())?; - let result = crate::query::duckdb_execute_for_database( - &locked, - &self.attached_names, - params.database.as_deref(), - ¶ms.sql, - params.max_rows, - )?; + let result = if params.preserve_insertion_order { + execute_with_preserved_insertion_order(&locked, &self.attached_names, ¶ms)? + } else { + crate::query::duckdb_execute_for_database( + &locked, + &self.attached_names, + params.database.as_deref(), + ¶ms.sql, + params.max_rows, + )? + }; if let Some(name) = crate::sql::attached_name_from_attach_sql(¶ms.sql) { if !self.attached_names.iter().any(|attached| attached.eq_ignore_ascii_case(&name)) { self.attached_names.push(name); @@ -191,6 +195,53 @@ impl DuckDbWorkerSession { } } +fn execute_with_preserved_insertion_order( + connection: &duckdb::Connection, + attached_names: &[String], + params: &DuckDbWorkerExecuteParams, +) -> Result { + let current: String = connection + .query_row("SELECT CAST(current_setting('preserve_insertion_order') AS VARCHAR)", [], |row| row.get(0)) + .map_err(|error| format!("Failed to read DuckDB preserve_insertion_order setting: {error}"))?; + let was_enabled = current + .parse::() + .map_err(|error| format!("Invalid DuckDB preserve_insertion_order setting {current:?}: {error}"))?; + if was_enabled { + return crate::query::duckdb_execute_for_database( + connection, + attached_names, + params.database.as_deref(), + ¶ms.sql, + params.max_rows, + ); + } + + // LIMIT/OFFSET Parquet pages are separate scans. Temporarily enabling this + // setting makes every page use the same source order without changing the + // user's connection-level setting after the request completes. + connection + .execute_batch("SET preserve_insertion_order = true") + .map_err(|error| format!("Failed to enable DuckDB insertion-order preservation: {error}"))?; + let result = crate::query::duckdb_execute_for_database( + connection, + attached_names, + params.database.as_deref(), + ¶ms.sql, + params.max_rows, + ); + let restore = connection.execute_batch("SET preserve_insertion_order = false"); + + match (result, restore) { + (result, Ok(())) => result, + (Ok(_), Err(error)) => { + Err(format!("DuckDB query succeeded but restoring preserve_insertion_order failed: {error}")) + } + (Err(query_error), Err(restore_error)) => { + Err(format!("{query_error}; restoring preserve_insertion_order also failed: {restore_error}")) + } + } +} + fn expand_tilde(path: &str) -> String { if !path.starts_with('~') { return path.to_string(); @@ -497,13 +548,69 @@ mod tests { .expect("connect"); let result = session - .execute(DuckDbWorkerExecuteParams { sql: "SELECT 1 AS value".to_string(), database: None, max_rows: None }) + .execute(DuckDbWorkerExecuteParams { + sql: "SELECT 1 AS value".to_string(), + database: None, + max_rows: None, + preserve_insertion_order: false, + }) .expect("execute"); assert_eq!(result.columns, vec!["value"]); assert_eq!(result.rows, vec![vec![serde_json::json!(1)]]); } + #[test] + fn worker_session_scopes_insertion_order_preservation() { + let mut session = DuckDbWorkerSession::default(); + session + .connect(DuckDbWorkerConnectParams { + path: ":memory:".to_string(), + attached_databases: Vec::new(), + init_script: Some("SET preserve_insertion_order = false".to_string()), + }) + .expect("connect"); + + let disabled = session + .execute(DuckDbWorkerExecuteParams { + sql: "SELECT current_setting('preserve_insertion_order') AS setting".to_string(), + database: None, + max_rows: None, + preserve_insertion_order: false, + }) + .expect("read initial setting"); + assert_eq!(disabled.rows, vec![vec![serde_json::json!(false)]]); + + let preserved = session + .execute(DuckDbWorkerExecuteParams { + sql: "SELECT current_setting('preserve_insertion_order') AS setting".to_string(), + database: None, + max_rows: None, + preserve_insertion_order: true, + }) + .expect("execute with order preservation"); + assert_eq!(preserved.rows, vec![vec![serde_json::json!(true)]]); + + session + .execute(DuckDbWorkerExecuteParams { + sql: "SELECT * FROM missing_table".to_string(), + database: None, + max_rows: None, + preserve_insertion_order: true, + }) + .expect_err("missing table should fail"); + + let restored = session + .execute(DuckDbWorkerExecuteParams { + sql: "SELECT current_setting('preserve_insertion_order') AS setting".to_string(), + database: None, + max_rows: None, + preserve_insertion_order: false, + }) + .expect("read restored setting"); + assert_eq!(restored.rows, vec![vec![serde_json::json!(false)]]); + } + #[test] fn worker_session_lists_tables_and_columns() { let mut session = DuckDbWorkerSession::default(); @@ -522,6 +629,7 @@ mod tests { .to_string(), database: None, max_rows: None, + preserve_insertion_order: false, }) .expect("create table"); @@ -566,6 +674,7 @@ mod tests { sql: "CREATE VIEW active_orders AS SELECT 1 AS id".to_string(), database: None, max_rows: None, + preserve_insertion_order: false, }) .expect("create view"); @@ -713,6 +822,7 @@ mod tests { sql: format!("ATTACH '{}' AS \"sales db\";", attached_path.to_string_lossy().replace('\'', "''")), database: None, max_rows: None, + preserve_insertion_order: false, }) .expect("attach sql"); diff --git a/agents/drivers/duckdb/src/wire.rs b/agents/drivers/duckdb/src/wire.rs index d276132a43..24e03a29f0 100644 --- a/agents/drivers/duckdb/src/wire.rs +++ b/agents/drivers/duckdb/src/wire.rs @@ -271,6 +271,9 @@ pub struct DuckDbWorkerExecuteParams { pub database: Option, #[serde(default)] pub max_rows: Option, + /// Temporarily preserve source row order while executing this query. + #[serde(default)] + pub preserve_insertion_order: bool, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] diff --git a/agents/drivers/duckdb/tests/duckdb_worker_process.rs b/agents/drivers/duckdb/tests/duckdb_worker_process.rs index 39d2c026fd..568d852aec 100644 --- a/agents/drivers/duckdb/tests/duckdb_worker_process.rs +++ b/agents/drivers/duckdb/tests/duckdb_worker_process.rs @@ -351,6 +351,7 @@ async fn worker_process_exits_when_stdin_closes_during_active_query() { sql: "SELECT sum(sin(i::DOUBLE) * cos(i::DOUBLE / 3.0)) FROM range(100000000000) AS t(i)".to_string(), database: None, max_rows: Some(10), + preserve_insertion_order: false, }, ) .expect("execute request"), @@ -508,10 +509,14 @@ async fn worker_shutdown_checkpoints_and_removes_wal() { let _ = std::fs::remove_file(&db_path); let _ = std::fs::remove_file(&wal_path); - let client = - DuckDbWorkerClient::open_with_executable(executable.clone(), db_path.to_string_lossy().to_string(), Vec::new(), None) - .await - .expect("worker process connects"); + let client = DuckDbWorkerClient::open_with_executable( + executable.clone(), + db_path.to_string_lossy().to_string(), + Vec::new(), + None, + ) + .await + .expect("worker process connects"); client .execute( None, diff --git a/apps/desktop/src/components/import/TableImportDialog.vue b/apps/desktop/src/components/import/TableImportDialog.vue index 57c6c9a8b9..2c4c371c8b 100644 --- a/apps/desktop/src/components/import/TableImportDialog.vue +++ b/apps/desktop/src/components/import/TableImportDialog.vue @@ -162,6 +162,7 @@ const formatOptions: Array<{ value: api.TableImportSourceFormat; icon: any; labe { value: "json", icon: FileJson, labelKey: "tableImport.formatJson", descriptionKey: "tableImport.formatJsonDescription" }, { value: "excel", icon: FileSpreadsheet, labelKey: "tableImport.formatExcel", descriptionKey: "tableImport.formatExcelDescription" }, { value: "sql", icon: FileCode, labelKey: "tableImport.formatSql", descriptionKey: "tableImport.formatSqlDescription" }, + { value: "parquet", icon: FileText, labelKey: "tableImport.formatParquet", descriptionKey: "tableImport.formatParquetDescription" }, ]; const encodingOptions = TABLE_IMPORT_ENCODING_OPTIONS; @@ -176,6 +177,9 @@ const wizardSteps: Array<{ value: TableImportWizardStep; labelKey: string }> = [ const selectedConnection = computed(() => (props.prefillConnectionId ? store.getConfig(props.prefillConnectionId) : undefined)); const structureDatabaseType = computed(() => tableStructureDatabaseTypeForConnection(selectedConnection.value)); +const supportsDuckDbParquetImport = computed(() => structureDatabaseType.value === "duckdb"); +const availableFormatOptions = computed(() => formatOptions.filter((format) => format.value !== "parquet" || supportsDuckDbParquetImport.value)); +const importFileExtensions = computed(() => ["csv", "tsv", "txt", "json", "xlsx", "xlsm", "xls", "sql", ...(supportsDuckDbParquetImport.value ? ["parquet"] : [])]); // Mirrors the conflict SQL dispatch in transfer.rs. Other dialects must keep // ordinary INSERT/error behavior instead of approximating an upsert. const IMPORT_CONFLICT_DATABASE_TYPES = new Set(["postgres", "kingbase", "opengauss", "sqlite", "cloudflare-d1", "duckdb", "mysql", "doris", "starrocks"]); @@ -426,6 +430,7 @@ function detectFormat(name: string): api.TableImportSourceFormat { if (lower.endsWith(".json")) return "json"; if (lower.endsWith(".xls") || lower.endsWith(".xlsx") || lower.endsWith(".xlsm")) return "excel"; if (lower.endsWith(".sql")) return "sql"; + if (lower.endsWith(".parquet")) return "parquet"; return "csv"; } @@ -671,6 +676,8 @@ async function loadTargetColumns() { async function previewSelectedImportFile(fileOrPath: string | File) { const input = importPreviewInput(uploadedImportSourceFromPreview(preview.value), fileOrPath); return api.previewTableImportFile(input.fileOrPath, { + connectionId: props.prefillConnectionId, + database: props.prefillDatabase || "", sourceRef: input.sourceRef, sourceFormat: sourceFormat.value, parseOptions: parseOptions.value, @@ -704,6 +711,13 @@ async function loadPreview(fileOrPath = selectedSource.value) { } function assignSelectedSource(source: string | File) { + const detectedFormat = detectFormat(typeof source === "string" ? source : source.name); + if (detectedFormat === "parquet" && !supportsDuckDbParquetImport.value) { + selectedSource.value = null; + preview.value = null; + errorMessage.value = t("tableImport.parquetOnlyDuckdb"); + return; + } batchTasks.value = []; selectedSource.value = source; preview.value = null; @@ -712,7 +726,7 @@ function assignSelectedSource(source: string | File) { progress.value = null; errorMessage.value = ""; const name = typeof source === "string" ? source : source.name; - sourceFormat.value = detectFormat(name); + sourceFormat.value = detectedFormat; emptyStringAsNull.value = defaultTableImportEmptyStringAsNull(sourceFormat.value); if (!newTableName.value.trim()) { newTableName.value = suggestedTableName(name); @@ -750,11 +764,18 @@ async function prepareBatchSources(sources: ImportSource[]) { const tasks: BatchImportTask[] = []; const usedNames = new Set(); const formats = sources.map((source) => detectFormat(sourceName(source))); + if (formats.includes("parquet") && !supportsDuckDbParquetImport.value) { + errorMessage.value = t("tableImport.parquetOnlyDuckdb"); + loadingPreview.value = false; + return; + } emptyStringAsNull.value = formats.length && formats.every((format) => format === formats[0]) ? defaultTableImportEmptyStringAsNull(formats[0]!) : true; try { for (const [index, source] of sources.entries()) { const format = formats[index]!; const initialPreview = await api.previewTableImportFile(source, { + connectionId: props.prefillConnectionId, + database: props.prefillDatabase || "", sourceFormat: format, parseOptions: taskParseOptions(format), previewLimit: Math.max(1, Number(previewLimit.value) || 50), @@ -765,6 +786,8 @@ async function prepareBatchSources(sources: ImportSource[]) { const effectiveSheetName = sheetName && sheetName === initialPreview.sheets?.[0] ? "" : sheetName; const taskPreview = effectiveSheetName ? await api.previewTableImportFile(input.fileOrPath, { + connectionId: props.prefillConnectionId, + database: props.prefillDatabase || "", sourceRef: input.sourceRef, sourceFormat: format, parseOptions: taskParseOptions(format, effectiveSheetName), @@ -809,11 +832,12 @@ async function selectFile() { const selected = await open({ multiple: targetMode.value === "create", filters: [ - { name: "Data files", extensions: ["csv", "tsv", "txt", "json", "xlsx", "xlsm", "xls", "sql"] }, + { name: "Data files", extensions: importFileExtensions.value }, { name: "Text", extensions: ["csv", "tsv", "txt"] }, { name: "JSON", extensions: ["json"] }, { name: "Excel", extensions: ["xlsx", "xlsm", "xls"] }, { name: "SQL", extensions: ["sql"] }, + ...(supportsDuckDbParquetImport.value ? [{ name: "Parquet", extensions: ["parquet"] }] : []), ], }); if (!selected) return; @@ -1156,6 +1180,8 @@ async function reloadBatchPreviewsForEncoding() { if (!isDelimitedFormat(task.format) && task.format !== "sql") continue; const input = importPreviewInput(uploadedImportSourceFromPreview(task.preview), task.source); const nextPreview = await api.previewTableImportFile(input.fileOrPath, { + connectionId: props.prefillConnectionId, + database: props.prefillDatabase || "", sourceRef: input.sourceRef, sourceFormat: task.format, parseOptions: taskParseOptions(task.format, task.sheetName), @@ -1254,7 +1280,7 @@ watch(rawProgressPercent, (percent) => {
- +
{{ t("tableImport.target") }} @@ -1329,8 +1355,15 @@ watch(rawProgressPercent, (percent) => { {{ t("tableImport.selectFile") }}
-
-