diff --git a/crates/dbx-core/src/connection.rs b/crates/dbx-core/src/connection.rs index 6d698d973b..3f023da665 100644 --- a/crates/dbx-core/src/connection.rs +++ b/crates/dbx-core/src/connection.rs @@ -48,7 +48,7 @@ pub enum PoolKind { Elasticsearch(db::elasticsearch_driver::EsClient), Agent(Arc>), ExternalTabular(Arc), - ExternalDriver { driver_id: String, config: ConnectionConfig, session: Arc }, + ExternalDriver { driver_id: String, config: Arc, session: Arc }, } pub struct AppState { @@ -130,7 +130,7 @@ impl AppState { let session = self.plugins.start_driver_session(driver_id).await?; let params = serde_json::json!({ "connection": config }); session.invoke::("connect", params).await?; - Ok(PoolKind::ExternalDriver { driver_id: driver_id.to_string(), config: config.clone(), session }) + Ok(PoolKind::ExternalDriver { driver_id: driver_id.to_string(), config: Arc::new(config.clone()), session }) } pub async fn get_or_create_pool(&self, connection_id: &str, database: Option<&str>) -> Result { diff --git a/crates/dbx-core/src/plugins.rs b/crates/dbx-core/src/plugins.rs index 5838c9a122..73ccce6fd2 100644 --- a/crates/dbx-core/src/plugins.rs +++ b/crates/dbx-core/src/plugins.rs @@ -239,9 +239,8 @@ impl PluginDriverSession { method: method.to_string(), params, }; - let line = serde_json::to_string(&request).map_err(|err| err.to_string())?; - process.stdin.write_all(line.as_bytes()).await.map_err(|err| err.to_string())?; - process.stdin.write_all(b"\n").await.map_err(|err| err.to_string())?; + let line = encode_plugin_request_line(&request)?; + process.stdin.write_all(&line).await.map_err(|err| err.to_string())?; process.stdin.flush().await.map_err(|err| err.to_string())?; let response_line = match read_plugin_line(&mut process.stdout, "response").await { @@ -284,11 +283,10 @@ where let request = PluginRequest { jsonrpc: "2.0", id: 1, driver: driver_id.to_string(), method: method.to_string(), params }; - let line = serde_json::to_string(&request).map_err(|err| err.to_string())?; + let line = encode_plugin_request_line(&request)?; let mut stdin = child.stdin.take().ok_or("Plugin stdin unavailable")?; - stdin.write_all(line.as_bytes()).await.map_err(|err| err.to_string())?; - stdin.write_all(b"\n").await.map_err(|err| err.to_string())?; + stdin.write_all(&line).await.map_err(|err| err.to_string())?; drop(stdin); let stdout = child.stdout.take().ok_or("Plugin stdout unavailable")?; @@ -332,6 +330,13 @@ where decode_plugin_response(plugin, request.id, &response_line) } +fn encode_plugin_request_line(request: &PluginRequest) -> Result, String> { + let mut bytes = Vec::new(); + serde_json::to_writer(&mut bytes, request).map_err(|err| err.to_string())?; + bytes.push(b'\n'); + Ok(bytes) +} + fn spawn_plugin_child(plugin: &InstalledPlugin) -> Result { let executable = plugin .manifest diff --git a/crates/dbx-core/src/query.rs b/crates/dbx-core/src/query.rs index 654923d34c..638084dd2a 100644 --- a/crates/dbx-core/src/query.rs +++ b/crates/dbx-core/src/query.rs @@ -539,7 +539,7 @@ pub async fn do_execute( let max_rows = options.max_rows; drop(connections); wait_for_query(cancel_token, async move { - let params = external_driver_query_params(&config, &sql, &database, schema.as_deref()); + let params = external_driver_query_params(config.as_ref(), &sql, &database, schema.as_deref()); session.invoke::("executeQuery", params).await }) .await diff --git a/crates/dbx-core/src/schema.rs b/crates/dbx-core/src/schema.rs index 2dcd8af4a3..ef23d3b945 100644 --- a/crates/dbx-core/src/schema.rs +++ b/crates/dbx-core/src/schema.rs @@ -240,7 +240,7 @@ pub async fn list_databases_core(state: &AppState, connection_id: &str) -> Resul let session = session.clone(); drop(connections); return session - .invoke::>("listDatabases", serde_json::json!({ "connection": config })) + .invoke::>("listDatabases", serde_json::json!({ "connection": config.as_ref() })) .await; } if let Some(client) = extract_clickhouse(&connections, connection_id) { @@ -298,7 +298,10 @@ pub async fn list_schemas_core(state: &AppState, connection_id: &str, database: let session = session.clone(); drop(connections); return session - .invoke::>("listSchemas", serde_json::json!({ "connection": config, "database": database })) + .invoke::>( + "listSchemas", + serde_json::json!({ "connection": config.as_ref(), "database": database }), + ) .await; } if let Some(client) = extract_sqlserver(&connections, &pool_key) { @@ -352,7 +355,7 @@ pub async fn list_tables_core( return session .invoke::>( "listTables", - serde_json::json!({ "connection": config, "database": database, "schema": schema }), + serde_json::json!({ "connection": config.as_ref(), "database": database, "schema": schema }), ) .await; } @@ -497,7 +500,7 @@ pub async fn list_objects_core( return session .invoke::>( "listObjects", - serde_json::json!({ "connection": config, "database": database, "schema": schema }), + serde_json::json!({ "connection": config.as_ref(), "database": database, "schema": schema }), ) .await; } @@ -574,7 +577,7 @@ pub async fn get_columns_core( .invoke::>( "getColumns", serde_json::json!({ - "connection": config, + "connection": config.as_ref(), "database": database, "schema": schema, "table": table, diff --git a/crates/dbx-core/src/storage.rs b/crates/dbx-core/src/storage.rs index 95173fdc28..fd7083eac7 100644 --- a/crates/dbx-core/src/storage.rs +++ b/crates/dbx-core/src/storage.rs @@ -805,10 +805,10 @@ impl Storage { async fn migrate_connections_json(&self, data_dir: &Path) -> Result<(), String> { let path = data_dir.join("connections.json"); - if !path.exists() { + if tokio::fs::metadata(&path).await.is_err() { return Ok(()); } - let json = std::fs::read_to_string(&path).map_err(|e| e.to_string())?; + let json = tokio::fs::read_to_string(&path).await.map_err(|e| e.to_string())?; let configs: Vec = serde_json::from_str(&json).unwrap_or_default(); for config in &configs { let config_json = serde_json::to_string(config).map_err(|e| e.to_string())?; @@ -819,16 +819,16 @@ impl Storage { .await .map_err(|e| e.to_string())?; } - std::fs::rename(&path, data_dir.join("connections.json.bak")).ok(); + let _ = tokio::fs::rename(&path, data_dir.join("connections.json.bak")).await; Ok(()) } async fn migrate_secrets_json(&self, data_dir: &Path) -> Result<(), String> { let path = data_dir.join("secrets.json"); - if !path.exists() { + if tokio::fs::metadata(&path).await.is_err() { return Ok(()); } - let json = std::fs::read_to_string(&path).map_err(|e| e.to_string())?; + let json = tokio::fs::read_to_string(&path).await.map_err(|e| e.to_string())?; let secrets: std::collections::HashMap = serde_json::from_str(&json).unwrap_or_default(); for (key, secret) in &secrets { // key format: "connection:{id}:{field}" @@ -846,16 +846,16 @@ impl Storage { .map_err(|e| e.to_string())?; } } - std::fs::rename(&path, data_dir.join("secrets.json.bak")).ok(); + let _ = tokio::fs::rename(&path, data_dir.join("secrets.json.bak")).await; Ok(()) } async fn migrate_history_json(&self, data_dir: &Path) -> Result<(), String> { let path = data_dir.join("query_history.json"); - if !path.exists() { + if tokio::fs::metadata(&path).await.is_err() { return Ok(()); } - let json = std::fs::read_to_string(&path).map_err(|e| e.to_string())?; + let json = tokio::fs::read_to_string(&path).await.map_err(|e| e.to_string())?; let entries: Vec = serde_json::from_str(&json).unwrap_or_default(); for entry in &entries { sqlx::query( @@ -884,16 +884,16 @@ impl Storage { .await .map_err(|e| e.to_string())?; } - std::fs::rename(&path, data_dir.join("query_history.json.bak")).ok(); + let _ = tokio::fs::rename(&path, data_dir.join("query_history.json.bak")).await; Ok(()) } async fn migrate_ai_config_json(&self, data_dir: &Path) -> Result<(), String> { let path = data_dir.join("ai_config.json"); - if !path.exists() { + if tokio::fs::metadata(&path).await.is_err() { return Ok(()); } - let json = std::fs::read_to_string(&path).map_err(|e| e.to_string())?; + let json = tokio::fs::read_to_string(&path).await.map_err(|e| e.to_string())?; // Only migrate if the table is empty let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM ai_config").fetch_one(&self.db).await.map_err(|e| e.to_string())?; @@ -904,16 +904,16 @@ impl Storage { .await .map_err(|e| e.to_string())?; } - std::fs::rename(&path, data_dir.join("ai_config.json.bak")).ok(); + let _ = tokio::fs::rename(&path, data_dir.join("ai_config.json.bak")).await; Ok(()) } async fn migrate_ai_conversations_json(&self, data_dir: &Path) -> Result<(), String> { let path = data_dir.join("ai_conversations.json"); - if !path.exists() { + if tokio::fs::metadata(&path).await.is_err() { return Ok(()); } - let json = std::fs::read_to_string(&path).map_err(|e| e.to_string())?; + let json = tokio::fs::read_to_string(&path).await.map_err(|e| e.to_string())?; let conversations: Vec = serde_json::from_str(&json).unwrap_or_default(); for conv in &conversations { let messages_json = serde_json::to_string(&conv.messages).map_err(|e| e.to_string())?; @@ -934,16 +934,16 @@ impl Storage { .await .map_err(|e| e.to_string())?; } - std::fs::rename(&path, data_dir.join("ai_conversations.json.bak")).ok(); + let _ = tokio::fs::rename(&path, data_dir.join("ai_conversations.json.bak")).await; Ok(()) } async fn migrate_sidebar_layout_json(&self, data_dir: &Path) -> Result<(), String> { let path = data_dir.join("sidebar_layout.json"); - if !path.exists() { + if tokio::fs::metadata(&path).await.is_err() { return Ok(()); } - let json = std::fs::read_to_string(&path).map_err(|e| e.to_string())?; + let json = tokio::fs::read_to_string(&path).await.map_err(|e| e.to_string())?; let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM sidebar_layout") .fetch_one(&self.db) .await @@ -955,7 +955,7 @@ impl Storage { .await .map_err(|e| e.to_string())?; } - std::fs::rename(&path, data_dir.join("sidebar_layout.json.bak")).ok(); + let _ = tokio::fs::rename(&path, data_dir.join("sidebar_layout.json.bak")).await; Ok(()) } } diff --git a/crates/dbx-core/src/table_import.rs b/crates/dbx-core/src/table_import.rs index 9025c2d8ab..287b9242fe 100644 --- a/crates/dbx-core/src/table_import.rs +++ b/crates/dbx-core/src/table_import.rs @@ -300,21 +300,26 @@ pub fn parse_xlsx_file(path: &str, preview_limit: usize) -> Result Result { +pub async fn parse_import_file(path: &str, preview_limit: usize) -> Result { match import_file_kind(path)? { ImportFileKind::Csv => { - let bytes = std::fs::read(path).map_err(|e| e.to_string())?; + let bytes = tokio::fs::read(path).await.map_err(|e| e.to_string())?; parse_csv_bytes(&bytes, preview_limit) } ImportFileKind::Tsv => { - let bytes = std::fs::read(path).map_err(|e| e.to_string())?; + let bytes = tokio::fs::read(path).await.map_err(|e| e.to_string())?; parse_delimited_bytes(&bytes, b'\t', preview_limit) } ImportFileKind::Json => { - let bytes = std::fs::read(path).map_err(|e| e.to_string())?; + let bytes = tokio::fs::read(path).await.map_err(|e| e.to_string())?; parse_json_bytes(&bytes, preview_limit) } - ImportFileKind::Xlsx => parse_xlsx_file(path, preview_limit), + ImportFileKind::Xlsx => { + let path = path.to_string(); + tokio::task::spawn_blocking(move || parse_xlsx_file(&path, preview_limit)) + .await + .map_err(|e| e.to_string())? + } } } @@ -394,10 +399,10 @@ pub fn truncate_sql(table: &str, schema: &str, db_type: &DatabaseType) -> String } } -pub fn preview_table_import_file_core(file_path: &str) -> Result { +pub async fn preview_table_import_file_core(file_path: &str) -> Result { let kind = import_file_kind(file_path)?; - let parsed = parse_import_file(file_path, DEFAULT_PREVIEW_LIMIT)?; - let metadata = std::fs::metadata(file_path).map_err(|e| e.to_string())?; + let parsed = parse_import_file(file_path, DEFAULT_PREVIEW_LIMIT).await?; + let metadata = tokio::fs::metadata(file_path).await.map_err(|e| e.to_string())?; let file_name = Path::new(file_path).file_name().and_then(|name| name.to_str()).unwrap_or(file_path).to_string(); Ok(TableImportPreview { @@ -426,7 +431,7 @@ where { let batch_size = if request.batch_size == 0 { DEFAULT_BATCH_SIZE } else { request.batch_size }; - let parsed = match parse_import_file(&request.file_path, usize::MAX) { + let parsed = match parse_import_file(&request.file_path, usize::MAX).await { Ok(parsed) => parsed, Err(error) => { progress_callback(TableImportProgress { diff --git a/crates/dbx-core/src/transfer.rs b/crates/dbx-core/src/transfer.rs index 93d25a5c91..71c5ddb734 100644 --- a/crates/dbx-core/src/transfer.rs +++ b/crates/dbx-core/src/transfer.rs @@ -353,20 +353,25 @@ pub fn generate_create_table_ddl( ) -> String { let full_table = qualified_table(table, schema, target_db); - let col_lines: Vec = columns - .iter() - .map(|c| { + let mut col_lines = Vec::with_capacity(columns.len()); + for c in columns { + col_lines.push({ let mapped_type = map_column_type(&c.data_type, source_db, target_db); let mut line = format!(" {} {}", quote_identifier(&c.name, target_db), mapped_type); if !c.is_nullable { line.push_str(" NOT NULL"); } line - }) - .collect(); + }); + } - let pks: Vec = - columns.iter().filter(|c| c.is_primary_key).map(|c| quote_identifier(&c.name, target_db)).collect(); + let mut pks = Vec::new(); + pks.reserve(columns.iter().filter(|c| c.is_primary_key).count()); + for c in columns { + if c.is_primary_key { + pks.push(quote_identifier(&c.name, target_db)); + } + } let mut ddl = match target_db { DatabaseType::SqlServer => { @@ -431,18 +436,15 @@ fn value_rows_sql( column_types: &[Option], db_type: &DatabaseType, ) -> Vec { - rows.iter() - .map(|row| { - let vals: Vec = row - .iter() - .enumerate() - .map(|(index, v)| { - escape_value_typed(v, db_type, column_types.get(index).and_then(|value| value.as_deref())) - }) - .collect(); - format!("({})", vals.join(", ")) - }) - .collect() + let mut out = Vec::with_capacity(rows.len()); + for row in rows { + let mut vals = Vec::with_capacity(row.len()); + for (index, v) in row.iter().enumerate() { + vals.push(escape_value_typed(v, db_type, column_types.get(index).and_then(|value| value.as_deref()))); + } + out.push(format!("({})", vals.join(", "))); + } + out } pub fn generate_upsert( @@ -474,7 +476,12 @@ pub fn generate_upsert_typed( let value_rows = value_rows_sql(rows, column_types, db_type); - let non_pk_columns: Vec<&String> = columns.iter().filter(|c| !pk_columns.contains(c)).collect(); + let mut non_pk_columns = Vec::with_capacity(columns.len().saturating_sub(pk_columns.len())); + for c in columns { + if !pk_columns.contains(c) { + non_pk_columns.push(c); + } + } match db_type { DatabaseType::Postgres | DatabaseType::Sqlite | DatabaseType::DuckDb => { @@ -549,28 +556,18 @@ pub fn generate_upsert_typed( sql } DatabaseType::Oracle => { - let using_rows: Vec = rows - .iter() - .map(|row| { - let vals: Vec = row - .iter() - .zip(columns.iter()) - .enumerate() - .map(|(index, (v, c))| { - format!( - "{} AS {}", - escape_value_typed( - v, - db_type, - column_types.get(index).and_then(|value| value.as_deref()) - ), - quote_identifier(c, db_type) - ) - }) - .collect(); - format!("SELECT {} FROM dual", vals.join(", ")) - }) - .collect(); + let mut using_rows = Vec::with_capacity(rows.len()); + for row in rows { + let mut vals = Vec::with_capacity(row.len().min(columns.len())); + for (index, (v, c)) in row.iter().zip(columns.iter()).enumerate() { + vals.push(format!( + "{} AS {}", + escape_value_typed(v, db_type, column_types.get(index).and_then(|value| value.as_deref())), + quote_identifier(c, db_type) + )); + } + using_rows.push(format!("SELECT {} FROM dual", vals.join(", "))); + } let on_clause = pk_columns .iter() diff --git a/crates/dbx-web/src/routes/table_import.rs b/crates/dbx-web/src/routes/table_import.rs index 5a87ea425d..594338feb3 100644 --- a/crates/dbx-web/src/routes/table_import.rs +++ b/crates/dbx-web/src/routes/table_import.rs @@ -42,8 +42,8 @@ pub async fn preview_import( std::fs::write(&file_path, &data).map_err(|e| AppError(e.to_string()))?; let file_path_str = file_path.to_string_lossy().to_string(); - let preview = table_import::preview_table_import_file_core(&file_path_str); - let _ = std::fs::remove_file(&file_path); + let preview = table_import::preview_table_import_file_core(&file_path_str).await; + let _ = tokio::fs::remove_file(&file_path).await; let preview = preview.map_err(AppError)?; return Ok(Json(serde_json::to_value(preview).map_err(|e| AppError(e.to_string()))?)); } diff --git a/src-tauri/src/commands/table_import.rs b/src-tauri/src/commands/table_import.rs index 86882da19c..b986c7bfc7 100644 --- a/src-tauri/src/commands/table_import.rs +++ b/src-tauri/src/commands/table_import.rs @@ -27,7 +27,7 @@ async fn clear_cancelled(import_id: &str) { #[tauri::command] pub async fn preview_table_import_file(file_path: String) -> Result { - dbx_core::table_import::preview_table_import_file_core(&file_path) + dbx_core::table_import::preview_table_import_file_core(&file_path).await } #[tauri::command]