fix(dameng): restore exports with identity columns

This commit is contained in:
t8y2 committed 2026-06-26 21:25:39 +08:00
1 parent fe03f740b7
commit 796b0ce5b7
9 files changed
+264 -11

No files matched your search

+59 -4
View File
@@ -6,7 +6,11 @@ use tokio::sync::RwLock;
use crate::models::connection::DatabaseType;
use crate::sql_dialect::{qualified_table_name, quote_table_identifier};
use crate::transfer::{format_ch_array_sql_literal, format_pg_array_sql_literal, quote_identifier};
use crate::transfer::{
format_ch_array_sql_literal, format_pg_array_sql_literal, is_identity_column_extra, quote_identifier,
selected_columns_include_identity_extras, wrap_dameng_identity_insert_sql,
wrap_dameng_identity_insert_sql_for_table,
};
static EXPORT_CANCELLED: std::sync::LazyLock<RwLock<HashSet<String>>> =
std::sync::LazyLock::new(|| RwLock::new(HashSet::new()));
@@ -89,6 +93,8 @@ pub struct ExportedTableSql {
#[serde(default)]
pub column_types: Vec<Option<String>>,
#[serde(default)]
pub column_extras: Vec<Option<String>>,
#[serde(default)]
pub rows: Vec<Vec<Value>>,
#[serde(default)]
pub truncated: bool,
@@ -110,6 +116,8 @@ pub struct BuildExportInsertStatementsOptions {
#[serde(default)]
pub column_types: Vec<Option<String>>,
#[serde(default)]
pub column_extras: Vec<Option<String>>,
#[serde(default)]
pub rows: Vec<Vec<Value>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub batch_size: Option<usize>,
@@ -254,6 +262,10 @@ pub fn build_export_insert_statements(options: BuildExportInsertStatementsOption
.collect::<Vec<_>>()
.join(", ");
let mut statements = Vec::new();
let needs_dameng_identity_insert = options.database_type == Some(DatabaseType::Dameng)
&& insert_columns.iter().any(|(index, _)| {
is_identity_column_extra(options.column_extras.get(*index).and_then(|value| value.as_deref()))
});
for rows in options.rows.chunks(batch_size) {
let values = rows
@@ -275,7 +287,12 @@ pub fn build_export_insert_statements(options: BuildExportInsertStatementsOption
})
.collect::<Vec<_>>()
.join(", ");
statements.push(format!("INSERT INTO {table} ({columns}) VALUES {values};"));
let insert_sql = format!("INSERT INTO {table} ({columns}) VALUES {values};");
if needs_dameng_identity_insert {
statements.push(wrap_dameng_identity_insert_sql_for_table(&insert_sql, &table));
} else {
statements.push(insert_sql);
}
}
Ok(statements)
@@ -329,6 +346,7 @@ pub fn build_database_sql_export(options: BuildDatabaseSqlExportOptions) -> Resu
qualified_table_name: table.qualified_table_name,
columns: table.columns,
column_types: table.column_types,
column_extras: table.column_extras,
rows: table.rows,
batch_size: Some(insert_batch_size),
})?;
@@ -766,6 +784,7 @@ pub async fn export_database_sql_core(
};
let col_names = columns.iter().map(|c| c.name.clone()).collect::<Vec<_>>();
let col_types = columns.iter().map(|c| Some(c.data_type.clone())).collect::<Vec<_>>();
let col_extras = columns.iter().map(|c| c.extra.clone()).collect::<Vec<_>>();
if !col_names.is_empty() {
// Get row count
@@ -822,7 +841,7 @@ pub async fn export_database_sql_core(
break;
}
let insert_sql = crate::transfer::generate_insert_typed(
let mut insert_sql = crate::transfer::generate_insert_typed(
&col_names,
&col_types,
&result.rows,
@@ -830,9 +849,18 @@ pub async fn export_database_sql_core(
&request.schema,
&db_type,
);
if db_type == DatabaseType::Dameng
&& selected_columns_include_identity_extras(&col_names, &col_extras)
{
insert_sql = wrap_dameng_identity_insert_sql(&insert_sql, table_name, &request.schema);
}
if !insert_sql.is_empty() {
writeln!(file, "{};\n", insert_sql).map_err(|e| format!("Failed to write file: {e}"))?;
if insert_sql.trim_end().ends_with(';') {
writeln!(file, "{}\n", insert_sql).map_err(|e| format!("Failed to write file: {e}"))?;
} else {
writeln!(file, "{};\n", insert_sql).map_err(|e| format!("Failed to write file: {e}"))?;
}
}
rows_exported += row_count as u64;
@@ -1106,6 +1134,7 @@ mod tests {
qualified_table_name: None,
columns: vec!["id".to_string(), "name".to_string()],
column_types: Vec::new(),
column_extras: Vec::new(),
rows: vec![vec![json!(1), json!("Ada")], vec![json!(2), json!("O'Hara")], vec![json!(3), json!("Linus")]],
batch_size: Some(2),
})
@@ -1129,6 +1158,7 @@ mod tests {
qualified_table_name: None,
columns: vec!["enabled".to_string(), "mask".to_string(), "label".to_string()],
column_types: vec![Some("bit(1)".to_string()), Some("BIT(4)".to_string()), Some("varchar(20)".to_string())],
column_extras: Vec::new(),
rows: vec![vec![json!("1"), json!("1010"), json!("1010")], vec![json!(false), json!(3), json!("off")]],
batch_size: Some(10),
})
@@ -1149,6 +1179,7 @@ mod tests {
qualified_table_name: None,
columns: vec!["id".to_string(), "title".to_string(), "search_vector".to_string()],
column_types: vec![Some("integer".to_string()), Some("text".to_string()), Some("tsvector".to_string())],
column_extras: Vec::new(),
rows: vec![vec![json!(1), json!("Hello"), json!("'hello':1A")]],
batch_size: Some(10),
})
@@ -1157,6 +1188,29 @@ mod tests {
assert_eq!(statements, vec!["INSERT INTO \"public\".\"articles\" (\"id\", \"title\") VALUES (1, 'Hello');"]);
}
#[test]
fn dameng_identity_export_inserts_enable_identity_insert() {
let statements = build_export_insert_statements(BuildExportInsertStatementsOptions {
database_type: Some(DatabaseType::Dameng),
schema: Some("SYSDBA".to_string()),
table_name: Some("USERS".to_string()),
qualified_table_name: None,
columns: vec!["ID".to_string(), "NAME".to_string()],
column_types: vec![Some("INT".to_string()), Some("VARCHAR(20)".to_string())],
column_extras: vec![Some("identity".to_string()), None],
rows: vec![vec![json!(1), json!("Ada")]],
batch_size: Some(10),
})
.unwrap();
assert_eq!(
statements,
vec![
"SET IDENTITY_INSERT \"SYSDBA\".\"USERS\" ON;\nINSERT INTO \"SYSDBA\".\"USERS\" (\"ID\", \"NAME\") VALUES (1, 'Ada');\nSET IDENTITY_INSERT \"SYSDBA\".\"USERS\" OFF;"
]
);
}
#[test]
fn builds_database_sql_export_with_ddl_before_data() {
let sql = build_database_sql_export(BuildDatabaseSqlExportOptions {
@@ -1171,6 +1225,7 @@ mod tests {
ddl: Some("CREATE TABLE `users` (`id` int);".to_string()),
columns: vec!["id".to_string()],
column_types: Vec::new(),
column_extras: Vec::new(),
rows: vec![vec![json!(1)]],
truncated: true,
}],
+8 -3
View File
@@ -435,6 +435,7 @@ async fn try_export_native_table_stream(
db_type: &DatabaseType,
col_names: &[String],
column_types: &[Option<String>],
column_extras: &[Option<String>],
primary_keys: &[String],
total_rows: Option<u64>,
row_limit: Option<usize>,
@@ -640,6 +641,7 @@ async fn try_export_native_table_stream(
qualified_table_name: None,
columns: col_names.to_vec(),
column_types: column_types.to_vec(),
column_extras: column_extras.to_vec(),
rows: std::mem::take(pending_rows),
batch_size: Some(SQL_INSERT_BATCH_SIZE),
})?;
@@ -761,10 +763,10 @@ pub async fn export_table_data_core(
// 3. Resolve columns. Data grid exports can provide columns/primary keys
// directly, which avoids expensive metadata round-trips on JDBC drivers.
let requested_columns = request.columns.as_ref().filter(|columns| !columns.is_empty());
let (col_names, column_types, primary_keys) = if let Some(requested_columns) = requested_columns {
let (col_names, column_types, column_extras, primary_keys) = if let Some(requested_columns) = requested_columns {
let primary_keys = request.primary_keys.clone().unwrap_or_default();
let column_types = request.column_types.clone().unwrap_or_default();
(requested_columns.clone(), column_types, primary_keys)
(requested_columns.clone(), column_types, Vec::new(), primary_keys)
} else {
let columns = crate::schema::get_columns_core(
state,
@@ -776,8 +778,9 @@ pub async fn export_table_data_core(
.await?;
let col_names: Vec<String> = columns.iter().map(|c| c.name.clone()).collect();
let column_types: Vec<Option<String>> = columns.iter().map(|c| Some(c.data_type.clone())).collect();
let column_extras: Vec<Option<String>> = columns.iter().map(|c| c.extra.clone()).collect();
let primary_keys: Vec<String> = columns.iter().filter(|c| c.is_primary_key).map(|c| c.name.clone()).collect();
(col_names, column_types, primary_keys)
(col_names, column_types, column_extras, primary_keys)
};
if col_names.is_empty() {
@@ -844,6 +847,7 @@ pub async fn export_table_data_core(
&db_type,
&col_names,
&column_types,
&column_extras,
&primary_keys,
total_rows,
row_limit,
@@ -1246,6 +1250,7 @@ pub async fn export_table_data_core(
qualified_table_name: None,
columns: col_names.clone(),
column_types: column_types.clone(),
column_extras: column_extras.clone(),
rows: result.rows.clone(),
batch_size: Some(100),
})?;
+143 -1
View File
@@ -154,6 +154,81 @@ fn is_postgres_identity_extra(extra: Option<&str>) -> bool {
})
}
pub(crate) fn is_identity_column_extra(extra: Option<&str>) -> bool {
extra.is_some_and(|value| {
let normalized = value.trim().to_ascii_lowercase();
normalized.contains("identity") || normalized.contains("auto_increment") || normalized.contains("autoincrement")
})
}
pub(crate) fn selected_columns_include_identity_extras(columns: &[String], column_extras: &[Option<String>]) -> bool {
columns
.iter()
.enumerate()
.any(|(index, _)| is_identity_column_extra(column_extras.get(index).and_then(|extra| extra.as_deref())))
}
fn selected_columns_include_identity_columns(columns: &[String], all_columns: &[db::ColumnInfo]) -> bool {
all_columns.iter().any(|column| {
is_identity_column_extra(column.extra.as_deref())
&& columns.iter().any(|name| name.eq_ignore_ascii_case(&column.name))
})
}
fn dameng_identity_insert_statement(table: &str, schema: &str, enabled: bool) -> String {
let full_table = qualified_table(table, schema, &DatabaseType::Dameng);
format!("SET IDENTITY_INSERT {full_table} {}", if enabled { "ON" } else { "OFF" })
}
pub(crate) fn wrap_dameng_identity_insert_sql(insert_sql: &str, table: &str, schema: &str) -> String {
let full_table = qualified_table(table, schema, &DatabaseType::Dameng);
wrap_dameng_identity_insert_sql_for_table(insert_sql, &full_table)
}
pub(crate) fn wrap_dameng_identity_insert_sql_for_table(insert_sql: &str, full_table: &str) -> String {
let trimmed = insert_sql.trim().trim_end_matches(';').trim();
format!(
"{};\n{};\n{};",
format!("SET IDENTITY_INSERT {full_table} ON"),
trimmed,
format!("SET IDENTITY_INSERT {full_table} OFF")
)
}
async fn execute_transfer_write_statement(
state: &AppState,
target_pool_key: &str,
sql: &str,
target_db_type: &DatabaseType,
table: &str,
schema: &str,
needs_identity_insert: bool,
) -> Result<(), String> {
if !needs_identity_insert || !matches!(target_db_type, DatabaseType::Dameng) {
execute_on_pool(state, target_pool_key, sql).await?;
return Ok(());
}
let enable_sql = dameng_identity_insert_statement(table, schema, true);
let disable_sql = dameng_identity_insert_statement(table, schema, false);
execute_on_pool(state, target_pool_key, &enable_sql)
.await
.map_err(|e| format!("Failed to enable Dameng IDENTITY_INSERT for {table}: {e}"))?;
let write_result = execute_on_pool(state, target_pool_key, sql).await;
let disable_result = execute_on_pool(state, target_pool_key, &disable_sql).await;
match (write_result, disable_result) {
(Ok(_), Ok(_)) => Ok(()),
(Err(write_error), Ok(_)) => Err(write_error),
(Ok(_), Err(disable_error)) => {
Err(format!("Failed to disable Dameng IDENTITY_INSERT for {table}: {disable_error}"))
}
(Err(write_error), Err(disable_error)) => {
Err(format!("{write_error}; also failed to disable Dameng IDENTITY_INSERT for {table}: {disable_error}"))
}
}
}
fn rewrite_postgres_schema_qualified_references(input: &str, source_schema: &str, target_schema: &str) -> String {
if source_schema.trim().is_empty() || source_schema == target_schema {
return input.to_string();
@@ -3260,6 +3335,22 @@ where
(request.mode.clone(), vec![])
};
let writes_dameng_identity_columns = if matches!(target_db_type, DatabaseType::Dameng) {
let target_columns = get_columns_for_transfer(
state,
target_pool_key,
&request.target_connection_id,
&request.target_database,
&request.target_schema,
&target_table,
)
.await
.unwrap_or_default();
selected_columns_include_identity_columns(&col_names, &target_columns)
} else {
false
};
// Transfer data in batches
let batch_size = if request.batch_size == 0 { 1000 } else { request.batch_size };
let mut offset: u64 = 0;
@@ -3297,7 +3388,17 @@ where
&pk_columns,
);
for (statement_index, batch_sql) in write_statements.iter().enumerate() {
execute_on_pool(state, target_pool_key, batch_sql).await.map_err(|e| {
execute_transfer_write_statement(
state,
target_pool_key,
batch_sql,
target_db_type,
&target_table,
&request.target_schema,
writes_dameng_identity_columns,
)
.await
.map_err(|e| {
let absolute_row = parse_mysql_row_error(&e).map(|row| offset + row);
match absolute_row {
Some(row) => format!(
@@ -3826,6 +3927,47 @@ mod tests {
assert!(error.contains("both map to 'orders'"));
}
#[test]
fn detects_identity_extras_for_selected_columns() {
assert!(selected_columns_include_identity_extras(
&[String::from("id"), String::from("name")],
&[Some(String::from("identity")), None],
));
assert!(selected_columns_include_identity_extras(
&[String::from("id")],
&[Some(String::from("auto_increment"))],
));
assert!(!selected_columns_include_identity_extras(
&[String::from("name")],
&[None, Some(String::from("identity"))],
));
}
#[test]
fn detects_selected_identity_columns_from_target_metadata() {
let target_columns = vec![
db::ColumnInfo { extra: Some("identity".to_string()), ..test_column("ID", "INT") },
test_column("NAME", "VARCHAR(20)"),
];
assert!(selected_columns_include_identity_columns(&[String::from("id")], &target_columns));
assert!(!selected_columns_include_identity_columns(&[String::from("name")], &target_columns));
}
#[test]
fn dameng_identity_insert_wrapper_quotes_schema_and_table() {
let sql = wrap_dameng_identity_insert_sql(
"INSERT INTO \"SYSDBA\".\"USERS\" (\"ID\") VALUES\n(1);",
"USERS",
"SYSDBA",
);
assert_eq!(
sql,
"SET IDENTITY_INSERT \"SYSDBA\".\"USERS\" ON;\nINSERT INTO \"SYSDBA\".\"USERS\" (\"ID\") VALUES\n(1);\nSET IDENTITY_INSERT \"SYSDBA\".\"USERS\" OFF;"
);
}
#[test]
fn mysql_create_table_includes_column_comments() {
let cols = vec![