diff --git a/crates/dbx-core/src/database_export.rs b/crates/dbx-core/src/database_export.rs index b746b803aa..fe409fe265 100644 --- a/crates/dbx-core/src/database_export.rs +++ b/crates/dbx-core/src/database_export.rs @@ -10,8 +10,9 @@ use crate::mysql_ddl_normalize::DdlNormalizeOptions; use crate::object_source_sql::build_export_object_source_sql; use crate::sql_dialect::{qualified_table_name, quote_table_identifier, uses_single_row_insert_statements}; use crate::transfer::{ - format_ch_array_sql_literal, format_pg_array_sql_literal, is_identity_column_extra, quote_identifier, - quote_postgres_string_literal, selected_columns_include_identity_extras, wrap_dameng_identity_insert_sql, + format_ch_array_sql_literal, format_pg_array_sql_literal, is_identity_column_extra, + is_mysql_generated_column_extra, quote_identifier, quote_postgres_string_literal, + selected_columns_include_identity_extras, wrap_dameng_identity_insert_sql, wrap_dameng_identity_insert_sql_for_table, }; use crate::types::ObjectSourceKind; @@ -796,11 +797,12 @@ pub fn build_export_insert_statements(options: BuildExportInsertStatementsOption .iter() .enumerate() .filter(|(index, column)| { - !is_internal_export_column(options.database_type, column) - && !is_postgres_tsvector_export_column( - options.database_type, - options.column_types.get(*index).and_then(|value| value.as_deref()), - ) + is_export_insert_column( + options.database_type, + column, + options.column_types.get(*index).and_then(|value| value.as_deref()), + options.column_extras.get(*index).and_then(|value| value.as_deref()), + ) }) .collect::>(); if insert_columns.is_empty() { @@ -870,6 +872,17 @@ fn is_postgres_tsvector_export_column(database_type: Option, colum .unwrap_or(false) } +fn is_export_insert_column( + database_type: Option, + column: &str, + column_type: Option<&str>, + column_extra: Option<&str>, +) -> bool { + !is_internal_export_column(database_type, column) + && !is_postgres_tsvector_export_column(database_type, column_type) + && !(database_type == Some(DatabaseType::Mysql) && is_mysql_generated_column_extra(column_extra)) +} + fn is_postgres_json_export_column(database_type: Option, column_type: Option<&str>) -> bool { database_type == Some(DatabaseType::Postgres) && column_type @@ -1283,9 +1296,59 @@ fn write_database_export_rows( schema: &str, db_type: &DatabaseType, ) -> Result<(), String> { - let mut insert_sql = - crate::transfer::generate_insert_typed(columns, column_types, rows, table, schema, db_type, None); - if *db_type == DatabaseType::Dameng && selected_columns_include_identity_extras(columns, column_extras) { + let insert_indices = columns + .iter() + .enumerate() + .filter(|(index, column)| { + is_export_insert_column( + Some(*db_type), + column, + column_types.get(*index).and_then(|value| value.as_deref()), + column_extras.get(*index).and_then(|value| value.as_deref()), + ) + }) + .map(|(index, _)| index) + .collect::>(); + if insert_indices.is_empty() { + return Ok(()); + } + let filtered_columns; + let filtered_column_types; + let filtered_column_extras; + let filtered_rows; + let (insert_columns, insert_column_types, insert_column_extras, insert_rows) = if insert_indices.len() + == columns.len() + { + (columns, column_types, column_extras, rows) + } else { + filtered_columns = insert_indices.iter().map(|index| columns[*index].clone()).collect::>(); + filtered_column_types = + insert_indices.iter().map(|index| column_types.get(*index).cloned().unwrap_or(None)).collect::>(); + filtered_column_extras = + insert_indices.iter().map(|index| column_extras.get(*index).cloned().unwrap_or(None)).collect::>(); + filtered_rows = rows + .iter() + .map(|row| insert_indices.iter().map(|index| row.get(*index).cloned().unwrap_or(Value::Null)).collect()) + .collect::>>(); + ( + filtered_columns.as_slice(), + filtered_column_types.as_slice(), + filtered_column_extras.as_slice(), + filtered_rows.as_slice(), + ) + }; + let mut insert_sql = crate::transfer::generate_insert_typed( + insert_columns, + insert_column_types, + insert_rows, + table, + schema, + db_type, + None, + ); + if *db_type == DatabaseType::Dameng + && selected_columns_include_identity_extras(insert_columns, insert_column_extras) + { insert_sql = wrap_dameng_identity_insert_sql(&insert_sql, table, schema); } if insert_sql.is_empty() { @@ -2172,10 +2235,10 @@ mod tests { format_export_table_ddl, generate_postgres_extension_ddl, generate_postgres_sequence_create_ddl, generate_postgres_sequence_owner_ddl, generate_postgres_sequence_setval_sql, is_postgres_extension_member_routine, mysql_database_export_preamble, normalize_export_table_ddl, - record_export_error, BuildDatabaseSqlExportOptions, BuildExportInsertStatementsOptions, - DatabaseExportObjectCounts, DatabaseExportRequest, DdlNormalizeOptions, ExportedTableSql, - PostgresExportExtension, PostgresExportSequence, PostgresExtensionMembers, DATABASE_EXPORT_INSERT_BATCH_SIZE, - DATABASE_EXPORT_ROW_LIMIT, + record_export_error, write_database_export_rows, BuildDatabaseSqlExportOptions, + BuildExportInsertStatementsOptions, DatabaseExportObjectCounts, DatabaseExportRequest, DdlNormalizeOptions, + ExportedTableSql, PostgresExportExtension, PostgresExportSequence, PostgresExtensionMembers, + DATABASE_EXPORT_INSERT_BATCH_SIZE, DATABASE_EXPORT_ROW_LIMIT, }; use crate::models::connection::DatabaseType; use crate::types::{ObjectInfo, ObjectSourceKind, TableInfo}; @@ -2965,6 +3028,97 @@ mod tests { assert_eq!(statements, vec!["INSERT INTO \"public\".\"articles\" (\"id\", \"title\") VALUES (1, 'Hello');"]); } + #[test] + fn mysql_generated_columns_are_omitted_from_sql_inserts_but_kept_in_ddl() { + let ddl = "CREATE TABLE `orders` (`id` bigint AUTO_INCREMENT, `quantity` int, `unit_price` decimal(10,2), `virtual_total` decimal(10,2) GENERATED ALWAYS AS ((`quantity` * `unit_price`)) VIRTUAL, `stored_total` decimal(10,2) GENERATED ALWAYS AS ((`quantity` * `unit_price`)) STORED);"; + let sql = build_database_sql_export(BuildDatabaseSqlExportOptions { + database_name: "shop".to_string(), + exported_at: Some("2026-07-30T00:00:00.000Z".to_string()), + tables: vec![ExportedTableSql { + display_name: "orders".to_string(), + database_type: Some(DatabaseType::Mysql), + schema: None, + table_name: Some("orders".to_string()), + qualified_table_name: None, + ddl: Some(ddl.to_string()), + columns: vec![ + "id".to_string(), + "quantity".to_string(), + "unit_price".to_string(), + "virtual_total".to_string(), + "stored_total".to_string(), + "created_at".to_string(), + ], + column_types: vec![ + Some("bigint".to_string()), + Some("int".to_string()), + Some("decimal(10,2)".to_string()), + Some("decimal(10,2)".to_string()), + Some("decimal(10,2)".to_string()), + Some("timestamp".to_string()), + ], + column_extras: vec![ + Some("auto_increment".to_string()), + None, + None, + Some("VIRTUAL GENERATED".to_string()), + Some("stored generated".to_string()), + Some("DEFAULT_GENERATED".to_string()), + ], + rows: vec![vec![json!(7), json!(2), json!(3.5), json!(7.0), json!(7.0), json!("2026-07-30 08:00:00")]], + truncated: false, + }], + row_limit_per_table: Some(DATABASE_EXPORT_ROW_LIMIT), + insert_batch_size: Some(DATABASE_EXPORT_INSERT_BATCH_SIZE), + connection_id: None, + database: None, + schema: None, + omit_auto_increment: false, + }) + .unwrap(); + + assert!(sql.contains(ddl)); + assert!(sql.contains( + "INSERT INTO `orders` (`id`, `quantity`, `unit_price`, `created_at`) VALUES (7, 2, 3.5, '2026-07-30 08:00:00');" + )); + assert!(!sql.contains("INSERT INTO `orders` (`id`, `quantity`, `unit_price`, `virtual_total`")); + } + + #[test] + fn mysql_database_export_file_rows_omit_generated_columns() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("orders.sql"); + let mut file = std::fs::File::create(&path).unwrap(); + + write_database_export_rows( + &mut file, + &[vec![json!(7), json!(2), json!(7.0), json!("2026-07-30 08:00:00")]], + &["id".to_string(), "quantity".to_string(), "virtual_total".to_string(), "created_at".to_string()], + &[ + Some("bigint".to_string()), + Some("int".to_string()), + Some("decimal(10,2)".to_string()), + Some("timestamp".to_string()), + ], + &[ + Some("auto_increment".to_string()), + None, + Some("VIRTUAL GENERATED".to_string()), + Some("DEFAULT_GENERATED".to_string()), + ], + "orders", + "shop", + &DatabaseType::Mysql, + ) + .unwrap(); + drop(file); + + assert_eq!( + std::fs::read_to_string(path).unwrap(), + "INSERT INTO `orders` (`id`, `quantity`, `created_at`) VALUES\n(7, 2, '2026-07-30 08:00:00');\n\n" + ); + } + #[test] fn dameng_identity_export_inserts_enable_identity_insert() { let statements = build_export_insert_statements(BuildExportInsertStatementsOptions { diff --git a/crates/dbx-core/src/table_export.rs b/crates/dbx-core/src/table_export.rs index 72ad5a43a0..49b8eed585 100644 --- a/crates/dbx-core/src/table_export.rs +++ b/crates/dbx-core/src/table_export.rs @@ -138,6 +138,25 @@ fn resolve_requested_export_columns( (resolved_columns, resolved_column_types, resolved_primary_keys) } +fn requested_export_needs_column_extras(database_type: DatabaseType, format: &str) -> bool { + database_type == DatabaseType::Mysql && format.eq_ignore_ascii_case("sql") +} + +fn resolve_requested_export_column_extras( + requested_columns: &[String], + table_columns: &[crate::db::ColumnInfo], +) -> Vec> { + requested_columns + .iter() + .map(|requested| { + table_columns + .iter() + .find(|column| column.name.eq_ignore_ascii_case(requested)) + .and_then(|column| column.extra.clone()) + }) + .collect() +} + fn write_json_row_object(writer: &mut W, columns: &[String], row: &[Value]) -> Result<(), String> { writer.write_all(b"{\n").map_err(|e| format!("Failed to write JSON: {e}"))?; let mut first = true; @@ -1107,7 +1126,20 @@ async fn export_table_data_core_inner( request.column_types.as_deref(), request.primary_keys.as_deref(), ); - (col_names, column_types, Vec::new(), primary_keys) + let column_extras = if requested_export_needs_column_extras(db_type, &request.format) { + let table_columns = crate::schema::get_columns_core( + state, + &request.connection_id, + &request.database, + request.schema.as_deref().unwrap_or(""), + &request.table_name, + ) + .await?; + resolve_requested_export_column_extras(&col_names, &table_columns) + } else { + Vec::new() + }; + (col_names, column_types, column_extras, primary_keys) } else { let columns = crate::schema::get_columns_core( state, @@ -2131,6 +2163,33 @@ mod tests { assert_eq!(resolved_mysql, mysql_columns); } + #[test] + fn requested_mysql_sql_export_resolves_generated_column_extras_only_for_sql() { + let table_columns = vec![ + crate::db::ColumnInfo { + name: "ID".to_string(), + extra: Some("auto_increment".to_string()), + ..Default::default() + }, + crate::db::ColumnInfo { + name: "virtual_total".to_string(), + extra: Some("VIRTUAL GENERATED".to_string()), + ..Default::default() + }, + ]; + let requested_columns = vec!["virtual_total".to_string(), "id".to_string(), "missing".to_string()]; + + assert!(requested_export_needs_column_extras(DatabaseType::Mysql, "SQL")); + for format in ["csv", "json", "xlsx"] { + assert!(!requested_export_needs_column_extras(DatabaseType::Mysql, format)); + } + assert!(!requested_export_needs_column_extras(DatabaseType::Postgres, "sql")); + assert_eq!( + resolve_requested_export_column_extras(&requested_columns, &table_columns), + vec![Some("VIRTUAL GENERATED".to_string()), Some("auto_increment".to_string()), None] + ); + } + #[test] fn agent_table_read_unsupported_detects_old_agent_errors() { assert!(is_agent_table_read_unsupported("Agent RPC error (-1): unknown method: start_table_read")); diff --git a/crates/dbx-core/src/transfer.rs b/crates/dbx-core/src/transfer.rs index 0ce0876936..b9966482d4 100644 --- a/crates/dbx-core/src/transfer.rs +++ b/crates/dbx-core/src/transfer.rs @@ -495,6 +495,20 @@ pub(crate) fn is_identity_column_extra(extra: Option<&str>) -> bool { }) } +pub(crate) fn is_mysql_generated_column_extra(extra: Option<&str>) -> bool { + extra.is_some_and(|value| { + let mut parts = value.split_whitespace(); + let Some(first) = parts.next() else { + return false; + }; + if first.eq_ignore_ascii_case("generated") { + return true; + } + matches!(first.to_ascii_lowercase().as_str(), "virtual" | "stored" | "persistent") + && parts.next().is_some_and(|part| part.eq_ignore_ascii_case("generated")) + }) +} + pub(crate) fn selected_columns_include_identity_extras(columns: &[String], column_extras: &[Option]) -> bool { columns .iter() @@ -523,6 +537,10 @@ fn is_sqlserver_non_insertable_transfer_column( && is_sqlserver_rowversion_type(&column.data_type) } +fn is_mysql_non_insertable_transfer_column(column: &db::ColumnInfo, source_db_type: &DatabaseType) -> bool { + *source_db_type == DatabaseType::Mysql && is_mysql_generated_column_extra(column.extra.as_deref()) +} + fn writable_transfer_columns( columns: &[db::ColumnInfo], source_db_type: &DatabaseType, @@ -530,7 +548,10 @@ fn writable_transfer_columns( ) -> Vec { columns .iter() - .filter(|column| !is_sqlserver_non_insertable_transfer_column(column, source_db_type, target_db_type)) + .filter(|column| { + !is_sqlserver_non_insertable_transfer_column(column, source_db_type, target_db_type) + && !is_mysql_non_insertable_transfer_column(column, source_db_type) + }) .cloned() .collect() } @@ -5342,6 +5363,49 @@ mod tests { assert_eq!(writable.iter().map(|column| column.name.as_str()).collect::>(), vec!["id", "name"]); } + #[test] + fn mysql_writable_transfer_columns_skip_only_generated_columns() { + let columns = vec![ + test_column("id", "int"), + db::ColumnInfo { extra: Some("DEFAULT_GENERATED".to_string()), ..test_column("created_at", "timestamp") }, + db::ColumnInfo { extra: Some("auto_increment".to_string()), ..test_column("sequence_id", "bigint") }, + db::ColumnInfo { + extra: Some("VIRTUAL GENERATED".to_string()), + ..test_column("virtual_total", "decimal(10,2)") + }, + db::ColumnInfo { extra: Some("stored generated".to_string()), ..test_column("stored_hash", "varchar(64)") }, + db::ColumnInfo { + extra: Some("PERSISTENT GENERATED".to_string()), + ..test_column("persistent_total", "decimal(10,2)") + }, + db::ColumnInfo { extra: Some("GENERATED ALWAYS".to_string()), ..test_column("explicit_generated", "int") }, + db::ColumnInfo { + extra: Some("on update CURRENT_TIMESTAMP".to_string()), + ..test_column("updated_at", "timestamp") + }, + ]; + + let writable = writable_transfer_columns(&columns, &DatabaseType::Mysql, &DatabaseType::Mysql); + + assert_eq!( + writable.iter().map(|column| column.name.as_str()).collect::>(), + vec!["id", "created_at", "sequence_id", "updated_at"] + ); + assert_eq!(columns.len(), 8, "DDL metadata must retain generated columns"); + } + + #[test] + fn non_mysql_transfer_columns_keep_generated_markers() { + let columns = vec![ + test_column("id", "int"), + db::ColumnInfo { extra: Some("STORED GENERATED".to_string()), ..test_column("computed", "int") }, + ]; + + let writable = writable_transfer_columns(&columns, &DatabaseType::Postgres, &DatabaseType::Postgres); + + assert_eq!(writable.iter().map(|column| column.name.as_str()).collect::>(), vec!["id", "computed"]); + } + #[test] fn non_sqlserver_target_writable_transfer_columns_keep_timestamp_type() { let columns = vec![test_column("id", "int"), test_column("updated_at", "timestamp")];