fix(import): allow selecting create-table column types

This commit is contained in:
t8y2 committed 2026-07-08 00:02:47 +08:00
1 parent 7eff6e544b
commit ad9841ab77
12 files changed
+494 -30

No files matched your search

+184 -22
View File
@@ -47,6 +47,8 @@ pub struct ImportCreateTablePlan {
pub struct TableImportColumnMapping {
pub source_column: String,
pub target_column: String,
#[serde(default)]
pub target_data_type: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -696,6 +698,15 @@ pub fn mapping_indexes_for_columns(
columns: &[String],
mappings: &[TableImportColumnMapping],
) -> Result<Vec<(usize, String)>, String> {
mapping_indexes_with_mappings(columns, mappings).map(|mapped| {
mapped.into_iter().map(|(source_index, mapping)| (source_index, mapping.target_column.clone())).collect()
})
}
fn mapping_indexes_with_mappings<'a>(
columns: &[String],
mappings: &'a [TableImportColumnMapping],
) -> Result<Vec<(usize, &'a TableImportColumnMapping)>, String> {
if mappings.is_empty() {
return Err("No columns mapped for import".to_string());
}
@@ -712,7 +723,7 @@ pub fn mapping_indexes_for_columns(
if !target_seen.insert(mapping.target_column.clone()) {
return Err(format!("Target column mapped more than once: {}", mapping.target_column));
}
mapped.push((source_index, mapping.target_column.clone()));
mapped.push((source_index, mapping));
}
Ok(mapped)
}
@@ -1011,6 +1022,52 @@ fn import_data_type(inferred_type: ImportInferredType, db_type: &DatabaseType) -
.to_string()
}
fn normalize_import_target_data_type(mapping: &TableImportColumnMapping) -> Result<Option<String>, String> {
let Some(raw_data_type) = mapping.target_data_type.as_deref() else {
return Ok(None);
};
let data_type = raw_data_type.trim();
if data_type.is_empty() {
return Err(format!("Target data type cannot be empty: {}", mapping.target_column));
}
validate_import_target_data_type(data_type)?;
Ok(Some(data_type.to_string()))
}
fn validate_import_target_data_type(data_type: &str) -> Result<(), String> {
let lowered = data_type.to_ascii_lowercase();
if data_type.contains(';')
|| lowered.contains("--")
|| lowered.contains("/*")
|| lowered.contains("*/")
|| data_type.chars().any(char::is_control)
{
return Err(format!("Unsupported target data type syntax: {data_type}"));
}
// A user-entered type is a DDL fragment, so keep it constrained to one type
// expression and reject separators that could add another column or clause.
let mut paren_depth = 0usize;
for ch in data_type.chars() {
match ch {
'(' => paren_depth += 1,
')' => {
paren_depth = paren_depth
.checked_sub(1)
.ok_or_else(|| format!("Unsupported target data type syntax: {data_type}"))?;
}
',' if paren_depth == 0 => {
return Err(format!("Unsupported target data type syntax: {data_type}"));
}
_ => {}
}
}
if paren_depth != 0 {
return Err(format!("Unsupported target data type syntax: {data_type}"));
}
Ok(())
}
pub fn build_import_create_table_plan(
data: &ParsedImportFile,
mappings: &[TableImportColumnMapping],
@@ -1021,12 +1078,17 @@ pub fn build_import_create_table_plan(
if table.trim().is_empty() {
return Err("Target table name is required".to_string());
}
let mapped = mapping_indexes(data, mappings)?;
let mapped = mapping_indexes_with_mappings(&data.columns, mappings)?;
let mut columns = Vec::with_capacity(mapped.len());
for (source_index, target_column) in mapped {
let inferred_type = infer_column_type(&data.rows, source_index);
columns
.push(ImportCreateTableColumn { name: target_column, data_type: import_data_type(inferred_type, db_type) });
for (source_index, mapping) in mapped {
let data_type = match normalize_import_target_data_type(mapping)? {
Some(data_type) => data_type,
None => {
let inferred_type = infer_column_type(&data.rows, source_index);
import_data_type(inferred_type, db_type)
}
};
columns.push(ImportCreateTableColumn { name: mapping.target_column.clone(), data_type });
}
if columns.is_empty() {
return Err("No columns mapped for import".to_string());
@@ -1694,7 +1756,11 @@ mod tests {
let mappings = data
.columns
.iter()
.map(|column| TableImportColumnMapping { source_column: column.clone(), target_column: column.clone() })
.map(|column| TableImportColumnMapping {
source_column: column.clone(),
target_column: column.clone(),
target_data_type: None,
})
.collect::<Vec<_>>();
let plan =
@@ -1721,8 +1787,11 @@ mod tests {
fn create_table_plan_requires_target_table_name() {
let data =
ParsedImportFile { columns: vec!["id".to_string()], rows: vec![vec![serde_json::json!(1)]], total_rows: 1 };
let mappings =
vec![TableImportColumnMapping { source_column: "id".to_string(), target_column: "id".to_string() }];
let mappings = vec![TableImportColumnMapping {
source_column: "id".to_string(),
target_column: "id".to_string(),
target_data_type: None,
}];
let error = build_import_create_table_plan(&data, &mappings, " ", "", &DatabaseType::Mysql).unwrap_err();
@@ -1736,19 +1805,80 @@ mod tests {
rows: vec![vec![serde_json::json!("long text")]],
total_rows: 1,
};
let mappings =
vec![TableImportColumnMapping { source_column: "notes".to_string(), target_column: "notes".to_string() }];
let mappings = vec![TableImportColumnMapping {
source_column: "notes".to_string(),
target_column: "notes".to_string(),
target_data_type: None,
}];
let plan = build_import_create_table_plan(&data, &mappings, "events", "dbo", &DatabaseType::SqlServer).unwrap();
assert_eq!(plan.sql, "CREATE TABLE [dbo].[events] (\n [notes] NVARCHAR(MAX)\n)");
}
#[test]
fn create_table_plan_uses_user_defined_column_type() {
let data = ParsedImportFile {
columns: vec!["code".to_string(), "amount".to_string()],
rows: vec![vec![serde_json::json!("1001"), serde_json::json!("12.5")]],
total_rows: 1,
};
let mappings = vec![
TableImportColumnMapping {
source_column: "code".to_string(),
target_column: "code".to_string(),
target_data_type: Some("VARCHAR(32)".to_string()),
},
TableImportColumnMapping {
source_column: "amount".to_string(),
target_column: "amount".to_string(),
target_data_type: Some("DECIMAL(10,2)".to_string()),
},
];
let plan = build_import_create_table_plan(&data, &mappings, "invoice", "", &DatabaseType::Mysql).unwrap();
assert_eq!(plan.sql, "CREATE TABLE `invoice` (\n `code` VARCHAR(32),\n `amount` DECIMAL(10,2)\n)");
assert_eq!(
plan.columns,
vec![
ImportCreateTableColumn { name: "code".to_string(), data_type: "VARCHAR(32)".to_string() },
ImportCreateTableColumn { name: "amount".to_string(), data_type: "DECIMAL(10,2)".to_string() },
]
);
}
#[test]
fn create_table_plan_rejects_unsafe_user_defined_column_type() {
let data = ParsedImportFile {
columns: vec!["name".to_string()],
rows: vec![vec![serde_json::json!("Ada")]],
total_rows: 1,
};
let mappings = vec![TableImportColumnMapping {
source_column: "name".to_string(),
target_column: "name".to_string(),
target_data_type: Some("TEXT, injected INT".to_string()),
}];
let error = build_import_create_table_plan(&data, &mappings, "users", "", &DatabaseType::Mysql).unwrap_err();
assert!(error.contains("Unsupported target data type syntax"));
}
#[test]
fn builds_import_insert_batches_from_mapped_columns() {
let mappings = vec![
TableImportColumnMapping { source_column: "id".to_string(), target_column: "user_id".to_string() },
TableImportColumnMapping { source_column: "name".to_string(), target_column: "display_name".to_string() },
TableImportColumnMapping {
source_column: "id".to_string(),
target_column: "user_id".to_string(),
target_data_type: None,
},
TableImportColumnMapping {
source_column: "name".to_string(),
target_column: "display_name".to_string(),
target_data_type: None,
},
];
let data = ParsedImportFile {
columns: vec!["id".to_string(), "name".to_string(), "ignored".to_string()],
@@ -1779,8 +1909,16 @@ mod tests {
fn duplicate_mapping_is_rejected_before_sql_generation() {
let columns = vec!["id".to_string(), "name".to_string()];
let mappings = vec![
TableImportColumnMapping { source_column: "id".to_string(), target_column: "target".to_string() },
TableImportColumnMapping { source_column: "name".to_string(), target_column: "target".to_string() },
TableImportColumnMapping {
source_column: "id".to_string(),
target_column: "target".to_string(),
target_data_type: None,
},
TableImportColumnMapping {
source_column: "name".to_string(),
target_column: "target".to_string(),
target_data_type: None,
},
];
let error = mapping_indexes_for_columns(&columns, &mappings).unwrap_err();
@@ -1792,8 +1930,16 @@ mod tests {
fn builds_single_streaming_import_batch_from_rows() {
let columns = vec!["id".to_string(), "name".to_string()];
let mappings = vec![
TableImportColumnMapping { source_column: "id".to_string(), target_column: "id".to_string() },
TableImportColumnMapping { source_column: "name".to_string(), target_column: "name".to_string() },
TableImportColumnMapping {
source_column: "id".to_string(),
target_column: "id".to_string(),
target_data_type: None,
},
TableImportColumnMapping {
source_column: "name".to_string(),
target_column: "name".to_string(),
target_data_type: None,
},
];
let rows = vec![vec![serde_json::json!(1), serde_json::json!("Ada")]];
@@ -1832,8 +1978,16 @@ mod tests {
#[test]
fn oracle_import_insert_batches_use_single_row_statements() {
let mappings = vec![
TableImportColumnMapping { source_column: "id".to_string(), target_column: "id".to_string() },
TableImportColumnMapping { source_column: "name".to_string(), target_column: "name".to_string() },
TableImportColumnMapping {
source_column: "id".to_string(),
target_column: "id".to_string(),
target_data_type: None,
},
TableImportColumnMapping {
source_column: "name".to_string(),
target_column: "name".to_string(),
target_data_type: None,
},
];
let data = ParsedImportFile {
columns: vec!["id".to_string(), "name".to_string()],
@@ -1873,8 +2027,13 @@ mod tests {
TableImportColumnMapping {
source_column: "start".to_string(),
target_column: "insurance_start_time".to_string(),
target_data_type: None,
},
TableImportColumnMapping {
source_column: "raw".to_string(),
target_column: "raw_text".to_string(),
target_data_type: None,
},
TableImportColumnMapping { source_column: "raw".to_string(), target_column: "raw_text".to_string() },
];
let data = ParsedImportFile {
columns: vec!["start".to_string(), "raw".to_string()],
@@ -1907,8 +2066,11 @@ mod tests {
#[test]
fn import_insert_batches_preserve_sqlserver_unicode_text() {
let mappings =
vec![TableImportColumnMapping { source_column: "name".to_string(), target_column: "name".to_string() }];
let mappings = vec![TableImportColumnMapping {
source_column: "name".to_string(),
target_column: "name".to_string(),
target_data_type: None,
}];
let data = ParsedImportFile {
columns: vec!["name".to_string()],
rows: vec![vec![serde_json::json!("Tiếng Việt")]],