diff --git a/agents/drivers/dameng/src/main/java/com/dbx/agent/dameng/DamengAgent.java b/agents/drivers/dameng/src/main/java/com/dbx/agent/dameng/DamengAgent.java index 959f8f26de..39d5d0858c 100644 --- a/agents/drivers/dameng/src/main/java/com/dbx/agent/dameng/DamengAgent.java +++ b/agents/drivers/dameng/src/main/java/com/dbx/agent/dameng/DamengAgent.java @@ -245,6 +245,7 @@ public final class DamengAgent extends BaseDatabaseAgent { } } + Set identityColumns = identityColumns(schema, table); List result = new ArrayList<>(); // DATA_DEFAULT is a LONG column — it must be selected first and read first // in JDBC, otherwise the data is truncated. @@ -287,7 +288,7 @@ public final class DamengAgent extends BaseDatabaseAgent { "Y".equals(rs.getString("NULLABLE")), dataDefault, pkColumns.contains(name), - null, + identityColumns.contains(name) ? "identity" : null, rs.getString("COMMENTS"), numPrec, numScale, @@ -301,6 +302,32 @@ public final class DamengAgent extends BaseDatabaseAgent { }); } + private Set identityColumns(String schema, String table) { + Set result = new java.util.HashSet<>(); + String sql = """ + SELECT c.NAME + FROM SYS.SYSCOLUMNS c + JOIN SYS.SYSOBJECTS t ON c.ID = t.ID + JOIN SYS.SYSOBJECTS s ON t.SCHID = s.ID + WHERE s.NAME = ? AND t.NAME = ? AND (c.INFO2 & 0x01) = 0x01 + """.stripIndent().trim(); + try (PreparedStatement stmt = requireConnected().prepareStatement(sql)) { + stmt.setString(1, schema); + stmt.setString(2, table); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + String column = rs.getString(1); + if (notBlank(column)) { + result.add(column); + } + } + } + } catch (Exception ignored) { + // Some Dameng versions or users do not expose SYS.SYSCOLUMNS. + } + return result; + } + @Override public List listIndexes(String schema, String table) { return unchecked(() -> { diff --git a/agents/drivers/dameng/src/test/java/com/dbx/agent/dameng/DamengAgentMetadataTest.java b/agents/drivers/dameng/src/test/java/com/dbx/agent/dameng/DamengAgentMetadataTest.java index fc79e2ba25..cb47a92bd5 100644 --- a/agents/drivers/dameng/src/test/java/com/dbx/agent/dameng/DamengAgentMetadataTest.java +++ b/agents/drivers/dameng/src/test/java/com/dbx/agent/dameng/DamengAgentMetadataTest.java @@ -109,6 +109,17 @@ class DamengAgentMetadataTest { Assertions.assertEquals("fallback id comment", columns.get(0).getComment()); } + @Test + void mapsIdentityColumnExtraFromSysColumns() { + DamengAgent agent = new DamengAgent(); + TestSupport.setPrivateConnection(agent, metadataConnection()); + + List columns = agent.getColumns("APP", "USERS"); + + Assertions.assertEquals(1, columns.size()); + Assertions.assertEquals("identity", columns.get(0).getExtra()); + } + @Test void appendsTableAndColumnCommentsToTableDdl() { DamengAgent agent = new DamengAgent(); @@ -136,6 +147,9 @@ class DamengAgentMetadataTest { if (sql.contains("ALL_CONS_COLUMNS")) { return metadataStatement(List.of(List.of("ID"))); } + if (sql.contains("SYS.SYSCOLUMNS")) { + return metadataStatement(List.of(List.of("ID"))); + } if (sql.startsWith("SELECT COMMENTS")) { return metadataStatement(List.of(List.of("用户示例表"))); } diff --git a/apps/desktop/src/lib/databaseExport.ts b/apps/desktop/src/lib/databaseExport.ts index 51880824b0..576350b279 100644 --- a/apps/desktop/src/lib/databaseExport.ts +++ b/apps/desktop/src/lib/databaseExport.ts @@ -16,6 +16,7 @@ export interface ExportedTableSql { ddl?: string; columns: string[]; columnTypes?: Array; + columnExtras?: Array; rows: QueryResult["rows"]; truncated?: boolean; } @@ -38,6 +39,7 @@ export interface BuildExportInsertStatementsOptions { qualifiedTableName?: string; columns: string[]; columnTypes?: Array; + columnExtras?: Array; rows: QueryResult["rows"]; batchSize?: number; } diff --git a/apps/desktop/src/lib/visibleDatabases.ts b/apps/desktop/src/lib/visibleDatabases.ts index 4f15b66535..eac1689033 100644 --- a/apps/desktop/src/lib/visibleDatabases.ts +++ b/apps/desktop/src/lib/visibleDatabases.ts @@ -52,7 +52,7 @@ const SYSTEM_DATABASE_RULES: Partial>> "xdb", "xs$null", ]), - dameng: new Set(["ctisys", "dba", "sys", "sysauditor", "sysdba", "syssso", "system"]), + dameng: new Set(["ctisys", "dba", "sys", "sysauditor", "syssso", "system"]), saphana: new Set(["_sys_afl", "_sys_bi", "_sys_bic", "_sys_repo", "_sys_statistics", "sys"]), cassandra: new Set(["system", "system_auth", "system_distributed", "system_schema", "system_traces", "system_views", "system_virtual_schema"]), neo4j: new Set(["system"]), diff --git a/crates/dbx-core/src/database_export.rs b/crates/dbx-core/src/database_export.rs index 56c9be58c4..1efa38cb32 100644 --- a/crates/dbx-core/src/database_export.rs +++ b/crates/dbx-core/src/database_export.rs @@ -6,7 +6,11 @@ use tokio::sync::RwLock; use crate::models::connection::DatabaseType; use crate::sql_dialect::{qualified_table_name, quote_table_identifier}; -use crate::transfer::{format_ch_array_sql_literal, format_pg_array_sql_literal, quote_identifier}; +use crate::transfer::{ + format_ch_array_sql_literal, format_pg_array_sql_literal, is_identity_column_extra, quote_identifier, + selected_columns_include_identity_extras, wrap_dameng_identity_insert_sql, + wrap_dameng_identity_insert_sql_for_table, +}; static EXPORT_CANCELLED: std::sync::LazyLock>> = std::sync::LazyLock::new(|| RwLock::new(HashSet::new())); @@ -89,6 +93,8 @@ pub struct ExportedTableSql { #[serde(default)] pub column_types: Vec>, #[serde(default)] + pub column_extras: Vec>, + #[serde(default)] pub rows: Vec>, #[serde(default)] pub truncated: bool, @@ -110,6 +116,8 @@ pub struct BuildExportInsertStatementsOptions { #[serde(default)] pub column_types: Vec>, #[serde(default)] + pub column_extras: Vec>, + #[serde(default)] pub rows: Vec>, #[serde(default, skip_serializing_if = "Option::is_none")] pub batch_size: Option, @@ -254,6 +262,10 @@ pub fn build_export_insert_statements(options: BuildExportInsertStatementsOption .collect::>() .join(", "); let mut statements = Vec::new(); + let needs_dameng_identity_insert = options.database_type == Some(DatabaseType::Dameng) + && insert_columns.iter().any(|(index, _)| { + is_identity_column_extra(options.column_extras.get(*index).and_then(|value| value.as_deref())) + }); for rows in options.rows.chunks(batch_size) { let values = rows @@ -275,7 +287,12 @@ pub fn build_export_insert_statements(options: BuildExportInsertStatementsOption }) .collect::>() .join(", "); - statements.push(format!("INSERT INTO {table} ({columns}) VALUES {values};")); + let insert_sql = format!("INSERT INTO {table} ({columns}) VALUES {values};"); + if needs_dameng_identity_insert { + statements.push(wrap_dameng_identity_insert_sql_for_table(&insert_sql, &table)); + } else { + statements.push(insert_sql); + } } Ok(statements) @@ -329,6 +346,7 @@ pub fn build_database_sql_export(options: BuildDatabaseSqlExportOptions) -> Resu qualified_table_name: table.qualified_table_name, columns: table.columns, column_types: table.column_types, + column_extras: table.column_extras, rows: table.rows, batch_size: Some(insert_batch_size), })?; @@ -766,6 +784,7 @@ pub async fn export_database_sql_core( }; let col_names = columns.iter().map(|c| c.name.clone()).collect::>(); let col_types = columns.iter().map(|c| Some(c.data_type.clone())).collect::>(); + let col_extras = columns.iter().map(|c| c.extra.clone()).collect::>(); if !col_names.is_empty() { // Get row count @@ -822,7 +841,7 @@ pub async fn export_database_sql_core( break; } - let insert_sql = crate::transfer::generate_insert_typed( + let mut insert_sql = crate::transfer::generate_insert_typed( &col_names, &col_types, &result.rows, @@ -830,9 +849,18 @@ pub async fn export_database_sql_core( &request.schema, &db_type, ); + if db_type == DatabaseType::Dameng + && selected_columns_include_identity_extras(&col_names, &col_extras) + { + insert_sql = wrap_dameng_identity_insert_sql(&insert_sql, table_name, &request.schema); + } if !insert_sql.is_empty() { - writeln!(file, "{};\n", insert_sql).map_err(|e| format!("Failed to write file: {e}"))?; + if insert_sql.trim_end().ends_with(';') { + writeln!(file, "{}\n", insert_sql).map_err(|e| format!("Failed to write file: {e}"))?; + } else { + writeln!(file, "{};\n", insert_sql).map_err(|e| format!("Failed to write file: {e}"))?; + } } rows_exported += row_count as u64; @@ -1106,6 +1134,7 @@ mod tests { qualified_table_name: None, columns: vec!["id".to_string(), "name".to_string()], column_types: Vec::new(), + column_extras: Vec::new(), rows: vec![vec![json!(1), json!("Ada")], vec![json!(2), json!("O'Hara")], vec![json!(3), json!("Linus")]], batch_size: Some(2), }) @@ -1129,6 +1158,7 @@ mod tests { qualified_table_name: None, columns: vec!["enabled".to_string(), "mask".to_string(), "label".to_string()], column_types: vec![Some("bit(1)".to_string()), Some("BIT(4)".to_string()), Some("varchar(20)".to_string())], + column_extras: Vec::new(), rows: vec![vec![json!("1"), json!("1010"), json!("1010")], vec![json!(false), json!(3), json!("off")]], batch_size: Some(10), }) @@ -1149,6 +1179,7 @@ mod tests { qualified_table_name: None, columns: vec!["id".to_string(), "title".to_string(), "search_vector".to_string()], column_types: vec![Some("integer".to_string()), Some("text".to_string()), Some("tsvector".to_string())], + column_extras: Vec::new(), rows: vec![vec![json!(1), json!("Hello"), json!("'hello':1A")]], batch_size: Some(10), }) @@ -1157,6 +1188,29 @@ mod tests { assert_eq!(statements, vec!["INSERT INTO \"public\".\"articles\" (\"id\", \"title\") VALUES (1, 'Hello');"]); } + #[test] + fn dameng_identity_export_inserts_enable_identity_insert() { + let statements = build_export_insert_statements(BuildExportInsertStatementsOptions { + database_type: Some(DatabaseType::Dameng), + schema: Some("SYSDBA".to_string()), + table_name: Some("USERS".to_string()), + qualified_table_name: None, + columns: vec!["ID".to_string(), "NAME".to_string()], + column_types: vec![Some("INT".to_string()), Some("VARCHAR(20)".to_string())], + column_extras: vec![Some("identity".to_string()), None], + rows: vec![vec![json!(1), json!("Ada")]], + batch_size: Some(10), + }) + .unwrap(); + + assert_eq!( + statements, + vec![ + "SET IDENTITY_INSERT \"SYSDBA\".\"USERS\" ON;\nINSERT INTO \"SYSDBA\".\"USERS\" (\"ID\", \"NAME\") VALUES (1, 'Ada');\nSET IDENTITY_INSERT \"SYSDBA\".\"USERS\" OFF;" + ] + ); + } + #[test] fn builds_database_sql_export_with_ddl_before_data() { let sql = build_database_sql_export(BuildDatabaseSqlExportOptions { @@ -1171,6 +1225,7 @@ mod tests { ddl: Some("CREATE TABLE `users` (`id` int);".to_string()), columns: vec!["id".to_string()], column_types: Vec::new(), + column_extras: Vec::new(), rows: vec![vec![json!(1)]], truncated: true, }], diff --git a/crates/dbx-core/src/table_export.rs b/crates/dbx-core/src/table_export.rs index d1678d5e46..5797f44b83 100644 --- a/crates/dbx-core/src/table_export.rs +++ b/crates/dbx-core/src/table_export.rs @@ -435,6 +435,7 @@ async fn try_export_native_table_stream( db_type: &DatabaseType, col_names: &[String], column_types: &[Option], + column_extras: &[Option], primary_keys: &[String], total_rows: Option, row_limit: Option, @@ -640,6 +641,7 @@ async fn try_export_native_table_stream( qualified_table_name: None, columns: col_names.to_vec(), column_types: column_types.to_vec(), + column_extras: column_extras.to_vec(), rows: std::mem::take(pending_rows), batch_size: Some(SQL_INSERT_BATCH_SIZE), })?; @@ -761,10 +763,10 @@ pub async fn export_table_data_core( // 3. Resolve columns. Data grid exports can provide columns/primary keys // directly, which avoids expensive metadata round-trips on JDBC drivers. let requested_columns = request.columns.as_ref().filter(|columns| !columns.is_empty()); - let (col_names, column_types, primary_keys) = if let Some(requested_columns) = requested_columns { + let (col_names, column_types, column_extras, primary_keys) = if let Some(requested_columns) = requested_columns { let primary_keys = request.primary_keys.clone().unwrap_or_default(); let column_types = request.column_types.clone().unwrap_or_default(); - (requested_columns.clone(), column_types, primary_keys) + (requested_columns.clone(), column_types, Vec::new(), primary_keys) } else { let columns = crate::schema::get_columns_core( state, @@ -776,8 +778,9 @@ pub async fn export_table_data_core( .await?; let col_names: Vec = columns.iter().map(|c| c.name.clone()).collect(); let column_types: Vec> = columns.iter().map(|c| Some(c.data_type.clone())).collect(); + let column_extras: Vec> = columns.iter().map(|c| c.extra.clone()).collect(); let primary_keys: Vec = columns.iter().filter(|c| c.is_primary_key).map(|c| c.name.clone()).collect(); - (col_names, column_types, primary_keys) + (col_names, column_types, column_extras, primary_keys) }; if col_names.is_empty() { @@ -844,6 +847,7 @@ pub async fn export_table_data_core( &db_type, &col_names, &column_types, + &column_extras, &primary_keys, total_rows, row_limit, @@ -1246,6 +1250,7 @@ pub async fn export_table_data_core( qualified_table_name: None, columns: col_names.clone(), column_types: column_types.clone(), + column_extras: column_extras.clone(), rows: result.rows.clone(), batch_size: Some(100), })?; diff --git a/crates/dbx-core/src/transfer.rs b/crates/dbx-core/src/transfer.rs index 6ff619a99f..0135462d41 100644 --- a/crates/dbx-core/src/transfer.rs +++ b/crates/dbx-core/src/transfer.rs @@ -154,6 +154,81 @@ fn is_postgres_identity_extra(extra: Option<&str>) -> bool { }) } +pub(crate) fn is_identity_column_extra(extra: Option<&str>) -> bool { + extra.is_some_and(|value| { + let normalized = value.trim().to_ascii_lowercase(); + normalized.contains("identity") || normalized.contains("auto_increment") || normalized.contains("autoincrement") + }) +} + +pub(crate) fn selected_columns_include_identity_extras(columns: &[String], column_extras: &[Option]) -> bool { + columns + .iter() + .enumerate() + .any(|(index, _)| is_identity_column_extra(column_extras.get(index).and_then(|extra| extra.as_deref()))) +} + +fn selected_columns_include_identity_columns(columns: &[String], all_columns: &[db::ColumnInfo]) -> bool { + all_columns.iter().any(|column| { + is_identity_column_extra(column.extra.as_deref()) + && columns.iter().any(|name| name.eq_ignore_ascii_case(&column.name)) + }) +} + +fn dameng_identity_insert_statement(table: &str, schema: &str, enabled: bool) -> String { + let full_table = qualified_table(table, schema, &DatabaseType::Dameng); + format!("SET IDENTITY_INSERT {full_table} {}", if enabled { "ON" } else { "OFF" }) +} + +pub(crate) fn wrap_dameng_identity_insert_sql(insert_sql: &str, table: &str, schema: &str) -> String { + let full_table = qualified_table(table, schema, &DatabaseType::Dameng); + wrap_dameng_identity_insert_sql_for_table(insert_sql, &full_table) +} + +pub(crate) fn wrap_dameng_identity_insert_sql_for_table(insert_sql: &str, full_table: &str) -> String { + let trimmed = insert_sql.trim().trim_end_matches(';').trim(); + format!( + "{};\n{};\n{};", + format!("SET IDENTITY_INSERT {full_table} ON"), + trimmed, + format!("SET IDENTITY_INSERT {full_table} OFF") + ) +} + +async fn execute_transfer_write_statement( + state: &AppState, + target_pool_key: &str, + sql: &str, + target_db_type: &DatabaseType, + table: &str, + schema: &str, + needs_identity_insert: bool, +) -> Result<(), String> { + if !needs_identity_insert || !matches!(target_db_type, DatabaseType::Dameng) { + execute_on_pool(state, target_pool_key, sql).await?; + return Ok(()); + } + + let enable_sql = dameng_identity_insert_statement(table, schema, true); + let disable_sql = dameng_identity_insert_statement(table, schema, false); + execute_on_pool(state, target_pool_key, &enable_sql) + .await + .map_err(|e| format!("Failed to enable Dameng IDENTITY_INSERT for {table}: {e}"))?; + let write_result = execute_on_pool(state, target_pool_key, sql).await; + let disable_result = execute_on_pool(state, target_pool_key, &disable_sql).await; + + match (write_result, disable_result) { + (Ok(_), Ok(_)) => Ok(()), + (Err(write_error), Ok(_)) => Err(write_error), + (Ok(_), Err(disable_error)) => { + Err(format!("Failed to disable Dameng IDENTITY_INSERT for {table}: {disable_error}")) + } + (Err(write_error), Err(disable_error)) => { + Err(format!("{write_error}; also failed to disable Dameng IDENTITY_INSERT for {table}: {disable_error}")) + } + } +} + fn rewrite_postgres_schema_qualified_references(input: &str, source_schema: &str, target_schema: &str) -> String { if source_schema.trim().is_empty() || source_schema == target_schema { return input.to_string(); @@ -3260,6 +3335,22 @@ where (request.mode.clone(), vec![]) }; + let writes_dameng_identity_columns = if matches!(target_db_type, DatabaseType::Dameng) { + let target_columns = get_columns_for_transfer( + state, + target_pool_key, + &request.target_connection_id, + &request.target_database, + &request.target_schema, + &target_table, + ) + .await + .unwrap_or_default(); + selected_columns_include_identity_columns(&col_names, &target_columns) + } else { + false + }; + // Transfer data in batches let batch_size = if request.batch_size == 0 { 1000 } else { request.batch_size }; let mut offset: u64 = 0; @@ -3297,7 +3388,17 @@ where &pk_columns, ); for (statement_index, batch_sql) in write_statements.iter().enumerate() { - execute_on_pool(state, target_pool_key, batch_sql).await.map_err(|e| { + execute_transfer_write_statement( + state, + target_pool_key, + batch_sql, + target_db_type, + &target_table, + &request.target_schema, + writes_dameng_identity_columns, + ) + .await + .map_err(|e| { let absolute_row = parse_mysql_row_error(&e).map(|row| offset + row); match absolute_row { Some(row) => format!( @@ -3826,6 +3927,47 @@ mod tests { assert!(error.contains("both map to 'orders'")); } + #[test] + fn detects_identity_extras_for_selected_columns() { + assert!(selected_columns_include_identity_extras( + &[String::from("id"), String::from("name")], + &[Some(String::from("identity")), None], + )); + assert!(selected_columns_include_identity_extras( + &[String::from("id")], + &[Some(String::from("auto_increment"))], + )); + assert!(!selected_columns_include_identity_extras( + &[String::from("name")], + &[None, Some(String::from("identity"))], + )); + } + + #[test] + fn detects_selected_identity_columns_from_target_metadata() { + let target_columns = vec![ + db::ColumnInfo { extra: Some("identity".to_string()), ..test_column("ID", "INT") }, + test_column("NAME", "VARCHAR(20)"), + ]; + + assert!(selected_columns_include_identity_columns(&[String::from("id")], &target_columns)); + assert!(!selected_columns_include_identity_columns(&[String::from("name")], &target_columns)); + } + + #[test] + fn dameng_identity_insert_wrapper_quotes_schema_and_table() { + let sql = wrap_dameng_identity_insert_sql( + "INSERT INTO \"SYSDBA\".\"USERS\" (\"ID\") VALUES\n(1);", + "USERS", + "SYSDBA", + ); + + assert_eq!( + sql, + "SET IDENTITY_INSERT \"SYSDBA\".\"USERS\" ON;\nINSERT INTO \"SYSDBA\".\"USERS\" (\"ID\") VALUES\n(1);\nSET IDENTITY_INSERT \"SYSDBA\".\"USERS\" OFF;" + ); + } + #[test] fn mysql_create_table_includes_column_comments() { let cols = vec![ diff --git a/crates/dbx-core/tests/live_postgres_tsvector.rs b/crates/dbx-core/tests/live_postgres_tsvector.rs index e12ccd59e6..0b1a566a72 100644 --- a/crates/dbx-core/tests/live_postgres_tsvector.rs +++ b/crates/dbx-core/tests/live_postgres_tsvector.rs @@ -84,6 +84,7 @@ async fn postgres_tsvector_generated_columns_are_readable_and_omitted_from_inser qualified_table_name: None, columns: result.columns.clone(), column_types: result.column_types.iter().map(|value| Some(value.clone())).collect(), + column_extras: Vec::new(), rows: result.rows.clone(), batch_size: Some(10), }) diff --git a/packages/app-tests/connectionVisibleDatabases.test.ts b/packages/app-tests/connectionVisibleDatabases.test.ts index e074c827d2..3237493f3a 100644 --- a/packages/app-tests/connectionVisibleDatabases.test.ts +++ b/packages/app-tests/connectionVisibleDatabases.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { test } from "vitest"; import { buildDraftVisibleDatabasesConnectionId, connectionCanChooseVisibleDatabases, visibleDatabaseSelectionIsStale, initialVisibleDatabaseSelection } from "../../apps/desktop/src/lib/connectionVisibleDatabases.ts"; -import { connectionUsesVisibleSchemaFilter } from "../../apps/desktop/src/lib/visibleDatabases.ts"; +import { connectionUsesVisibleSchemaFilter, filterDatabaseNamesForConnection } from "../../apps/desktop/src/lib/visibleDatabases.ts"; import type { ConnectionConfig } from "../../apps/desktop/src/types/database.ts"; function config(overrides: Partial = {}): ConnectionConfig { @@ -58,6 +58,13 @@ test("OceanBase Oracle uses schema filtering for visible object selection", () = assert.equal(connectionUsesVisibleSchemaFilter(config({ db_type: "mysql", driver_profile: "oceanbase" })), false); }); +test("Dameng default SYSDBA user remains selectable", () => { + assert.deepEqual( + filterDatabaseNamesForConnection(["SYS", "SYSDBA", "SYSAUDITOR"], config({ db_type: "dameng" })), + ["SYSDBA"], + ); +}); + test("visible database selection is stale when connection target changes", () => { const previous = config({ host: "db.internal", visible_databases: ["app"] }); assert.equal(visibleDatabaseSelectionIsStale(previous, config({ host: "db.internal" })), false);