feat(data-grid): add multi-format copy extractors

This commit is contained in:
soddygo authored and GitHub committed 2026-07-27 16:11:53 +08:00
1 parent 166027600f
commit 0b7a886cfd
60 files changed
+4615 -1766

No files matched your search

+58 -33
View File
@@ -30,6 +30,7 @@ const MYSQL_DATA_GRID_BATCH_MAX_ROWS: usize = 500;
const MYSQL_DATA_GRID_BATCH_TARGET_SQL_BYTES: usize = 256 * 1024;
#[derive(Debug, Clone, Serialize, Deserialize)]
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
#[serde(rename_all = "camelCase")]
pub struct DataGridTableMeta {
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -48,6 +49,7 @@ pub struct DataGridTableMeta {
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
pub struct DataGridColumnInfo {
pub name: String,
#[serde(default)]
@@ -113,10 +115,13 @@ pub struct DataGridCopyInsertStatementOptions {
#[serde(default)]
pub exclude_primary_keys: bool,
#[serde(default)]
pub include_computed_columns: bool,
#[serde(default)]
pub insert_mode: DataGridCopyInsertMode,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
#[serde(rename_all = "kebab-case")]
pub enum DataGridCopyInsertMode {
#[default]
@@ -289,8 +294,14 @@ pub fn prepare_data_grid_save(options: DataGridSaveStatementOptions) -> DataGrid
}
}
/// Relational SQL UPDATE/WHERE predicates are not meaningful for graph,
/// document, or time-series stores that don't speak relational SQL.
pub(crate) fn supports_relational_copy_predicates(database_type: Option<DatabaseType>) -> bool {
!matches!(database_type, Some(DatabaseType::Neo4j | DatabaseType::Tdengine | DatabaseType::MongoDb))
}
pub fn build_data_grid_copy_update_statements(options: DataGridCopyUpdateStatementOptions) -> Vec<String> {
if matches!(options.database_type, Some(DatabaseType::Neo4j | DatabaseType::Tdengine)) {
if !supports_relational_copy_predicates(options.database_type) {
return Vec::new();
}
let primary_keys = &options.table_meta.primary_keys;
@@ -310,13 +321,16 @@ pub fn build_data_grid_copy_update_statements(options: DataGridCopyUpdateStateme
let primary_key_indexes: Vec<usize> = primary_key_indexes.into_iter().flatten().collect();
let primary_key_set: Vec<String> =
primary_keys.iter().map(|primary_key| normalize_column_name(primary_key)).collect();
let writable_indexes: Vec<(&str, usize)> = save_columns
let writable_indexes: Vec<(&str, usize, Option<&DataGridColumnInfo>)> = save_columns
.iter()
.enumerate()
.filter_map(|(index, column)| Some((column.as_deref()?, index)))
.filter(|(column, _)| !primary_key_set.contains(&normalize_column_name(column)))
.filter(|(column, _)| !is_oracle_row_id(options.database_type, Some(column)))
.map(|(column, index)| (column, index, column_info_for(column_info, column)))
.collect();
let primary_key_info =
primary_keys.iter().map(|primary_key| column_info_for(column_info, primary_key)).collect::<Vec<_>>();
if writable_indexes.is_empty() {
return Vec::new();
@@ -337,15 +351,11 @@ pub fn build_data_grid_copy_update_statements(options: DataGridCopyUpdateStateme
}
let sets = writable_indexes
.iter()
.map(|(column, index)| {
.map(|(column, index, info)| {
format!(
"{} = {}",
data_grid_identifier(options.database_type, column, None),
format_grid_sql_literal(
row.get(*index).unwrap_or(&Value::Null),
options.database_type,
column_info_for(column_info, column)
)
format_grid_sql_literal(row.get(*index).unwrap_or(&Value::Null), options.database_type, *info)
)
})
.collect::<Vec<_>>()
@@ -361,7 +371,7 @@ pub fn build_data_grid_copy_update_statements(options: DataGridCopyUpdateStateme
options.database_type,
primary_key,
row.get(primary_key_indexes[index]).unwrap_or(&Value::Null),
column_info_for(column_info, primary_key),
primary_key_info[index],
false,
None,
)
@@ -384,20 +394,30 @@ pub fn build_data_grid_copy_insert_statement(options: DataGridCopyInsertStatemen
.as_ref()
.map(|meta| meta.primary_keys.iter().map(|primary_key| normalize_column_name(primary_key)).collect())
.unwrap_or_default();
let insertable_columns: Vec<(&str, usize)> = save_columns
let insertable_columns: Vec<(&str, usize, Option<DataGridColumnInfo>)> = save_columns
.iter()
.enumerate()
.filter_map(|(index, column)| Some((column.as_deref()?, index)))
.filter(|(column, _)| {
!is_grid_insert_omitted_column(options.database_type, column_info_for(column_info, column), Some(column))
.map(|(column, index)| {
let fallback_type =
options.column_types.as_deref().and_then(|types| types.get(index)).and_then(|value| value.as_deref());
(column, index, copy_column_info(column_info, column, fallback_type))
})
.filter(|(column, _, info)| {
!is_grid_insert_omitted_column(
options.database_type,
info.as_ref(),
Some(column),
options.include_computed_columns,
)
})
.collect();
let insert_columns: Vec<(&str, usize)> = insertable_columns
let insert_columns: Vec<(&str, usize, Option<DataGridColumnInfo>)> = insertable_columns
.iter()
.copied()
.filter(|(column, _)| {
.filter(|(column, _, _)| {
!options.exclude_primary_keys || !primary_key_set.contains(&normalize_column_name(column))
})
.cloned()
.collect();
if insert_columns.is_empty() || options.rows.is_empty() {
@@ -418,7 +438,7 @@ pub fn build_data_grid_copy_insert_statement(options: DataGridCopyInsertStatemen
);
let columns = insert_columns
.iter()
.map(|(column, _)| quote_ident(options.database_type, column))
.map(|(column, _, _)| quote_ident(options.database_type, column))
.collect::<Vec<_>>()
.join(", ");
let value_rows = options
@@ -429,20 +449,11 @@ pub fn build_data_grid_copy_insert_statement(options: DataGridCopyInsertStatemen
"({})",
insert_columns
.iter()
.map(|(column, index)| {
.map(|(_, index, info)| {
format_grid_copy_insert_sql_literal(
row.get(*index).unwrap_or(&Value::Null),
options.database_type,
copy_column_info(
column_info,
column,
options
.column_types
.as_deref()
.and_then(|types| types.get(*index))
.and_then(|value| value.as_deref()),
)
.as_ref(),
info.as_ref(),
)
})
.collect::<Vec<_>>()
@@ -1184,6 +1195,7 @@ fn build_data_grid_save_statements(options: &DataGridSaveStatementOptions) -> Ve
options.database_type,
column_info_for(column_info, column),
Some(column),
false,
)
})
.filter(|(_, value)| !value.is_null())
@@ -1280,6 +1292,7 @@ fn build_data_grid_rollback_statements(options: &DataGridSaveStatementOptions) -
options.database_type,
column_info_for(column_info, column),
Some(column),
false,
)
})
.collect();
@@ -1724,7 +1737,10 @@ pub fn format_grid_sql_literal(
// is a numeric/boolean type rather than a bit-string type like
// PostgreSQL's bit(n).
if let Some(value) = value.as_bool() {
if is_bit_literal_column(database_type, column_info) {
// SQL Server has no TRUE/FALSE literals (its boolean type is BIT, which
// is_bit_literal_column already covers); any other column there still
// needs numeric 1/0 instead of a literal.
if is_bit_literal_column(database_type, column_info) || database_type == Some(DatabaseType::SqlServer) {
return if value { "1" } else { "0" }.to_string();
}
return if value { "TRUE" } else { "FALSE" }.to_string();
@@ -2351,7 +2367,7 @@ fn build_save_row_where(
.join(" AND ")
}
fn build_column_predicate(
pub(crate) fn build_column_predicate(
database_type: Option<DatabaseType>,
column: &str,
value: &Value,
@@ -2479,7 +2495,7 @@ pub(crate) fn is_neo4j_element_id(database_type: Option<DatabaseType>, name: Opt
database_type == Some(DatabaseType::Neo4j) && name == Some(DBX_NEO4J_ELEMENT_ID_COLUMN)
}
fn is_auto_generated_column(column: &DataGridColumnInfo) -> bool {
pub(crate) fn is_auto_generated_column(column: &DataGridColumnInfo) -> bool {
column
.extra
.as_deref()
@@ -2493,14 +2509,15 @@ fn grid_value_is_empty(value: &Value) -> bool {
value.is_null() || value.as_str().is_some_and(str::is_empty)
}
fn is_grid_insert_omitted_column(
pub(crate) fn is_grid_insert_omitted_column(
database_type: Option<DatabaseType>,
column_info: Option<&DataGridColumnInfo>,
name: Option<&str>,
include_computed_columns: bool,
) -> bool {
is_oracle_row_id(database_type, name)
|| is_postgres_tsvector_column(database_type, column_info)
|| is_non_identity_generated_column(column_info)
|| (!include_computed_columns && is_non_identity_generated_column(column_info))
}
fn is_grid_update_omitted_column(
@@ -2548,7 +2565,7 @@ fn is_postgres_tsvector_type(data_type: &str) -> bool {
normalized == "tsvector" || normalized.ends_with(".tsvector")
}
fn is_non_identity_generated_column(column_info: Option<&DataGridColumnInfo>) -> bool {
pub(crate) fn is_non_identity_generated_column(column_info: Option<&DataGridColumnInfo>) -> bool {
let extra = column_info.and_then(|column| column.extra.as_deref()).unwrap_or("").to_ascii_lowercase();
extra.contains("generated always as") && !extra.contains("identity")
}
@@ -2941,6 +2958,7 @@ mod tests {
source_columns: None,
rows: vec![vec![json!(1), json!("ada"), json!("Ada")], vec![json!(2), json!("linus"), json!("Linus")]],
exclude_primary_keys: true,
include_computed_columns: false,
insert_mode: DataGridCopyInsertMode::Merged,
});
assert_eq!(
@@ -2966,6 +2984,7 @@ mod tests {
source_columns: None,
rows: vec![vec![json!("ada"), json!("Ada")]],
exclude_primary_keys: true,
include_computed_columns: false,
insert_mode: DataGridCopyInsertMode::Merged,
});
@@ -2992,6 +3011,7 @@ mod tests {
source_columns: None,
rows: vec![vec![json!(1), json!("ada"), json!("Ada")], vec![json!(2), json!("linus"), json!("Linus")]],
exclude_primary_keys: false,
include_computed_columns: false,
insert_mode: DataGridCopyInsertMode::RowByRow,
});
assert_eq!(
@@ -3019,6 +3039,7 @@ mod tests {
source_columns: None,
rows: vec![vec![json!(1), json!("Ada")], vec![json!(2), json!("Linus")]],
exclude_primary_keys: false,
include_computed_columns: false,
insert_mode: DataGridCopyInsertMode::Merged,
});
@@ -3049,6 +3070,7 @@ mod tests {
source_columns: None,
rows: rows.clone(),
exclude_primary_keys: false,
include_computed_columns: false,
insert_mode: DataGridCopyInsertMode::Merged,
});
assert_eq!(
@@ -3113,6 +3135,7 @@ mod tests {
source_columns: None,
rows: vec![vec![json!(1), json!("Hello"), json!("'hello':1A")]],
exclude_primary_keys: false,
include_computed_columns: false,
insert_mode: DataGridCopyInsertMode::Merged,
});
@@ -3136,6 +3159,7 @@ mod tests {
source_columns: None,
rows: vec![vec![json!(1), json!("2022-08-25T09:58:43Z"), json!("2022-08-25T09:58:43Z")]],
exclude_primary_keys: false,
include_computed_columns: false,
insert_mode: DataGridCopyInsertMode::Merged,
});
@@ -5643,6 +5667,7 @@ mod tests {
source_columns: None,
rows: vec![vec![json!(2), json!("new")]],
exclude_primary_keys: false,
include_computed_columns: false,
insert_mode: DataGridCopyInsertMode::Merged,
});
assert_eq!(