feat(import): update rows on primary-key conflicts

Closes #10014
This commit is contained in:
t8y2 committed 2026-09-27 05:54:32 +00:00
1 parent d5dcdabd22
commit 5f8bcf74a5
12 files changed
+758 -110

No files matched your search

@@ -406,6 +406,7 @@ fn import_request(
date_time_format: None,
prepared_source: None,
retain_source: false,
conflict_policy: None,
skip_duplicate_rows: false,
}
}
+466 -88
View File
@@ -29,7 +29,8 @@ use crate::sql::SqlParsingOptions;
use crate::sql_file_import::{SqlFileStreamDecoder, StreamingSqlFileSplitter};
use crate::transfer::{
escape_value_typed, execute_on_pool, generate_insert_typed_from_value_rows, get_columns_for_transfer,
normalize_integer_literal, normalize_thousands_numeric_literal, qualified_table, quote_identifier, SqlBatchLimits,
normalize_integer_literal, normalize_thousands_numeric_literal, qualified_table, quote_identifier,
supports_primary_key_upsert, SqlBatchLimits,
};
pub const DEFAULT_PREVIEW_LIMIT: usize = 50;
@@ -245,6 +246,15 @@ pub struct TableImportPreviewRequest {
pub preview_limit: Option<usize>,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub enum TableImportConflictPolicy {
#[default]
Error,
Skip,
UpdateExisting,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct TableImportRequest {
@@ -271,10 +281,83 @@ pub struct TableImportRequest {
pub prepared_source: Option<TableImportPreparedSource>,
#[serde(default)]
pub retain_source: bool,
/// Explicit conflict behavior for new callers. When omitted, the legacy
/// `skip_duplicate_rows` flag remains authoritative for compatibility.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub conflict_policy: Option<TableImportConflictPolicy>,
#[serde(default)]
pub skip_duplicate_rows: bool,
}
impl TableImportRequest {
fn effective_conflict_policy(&self) -> TableImportConflictPolicy {
self.conflict_policy.unwrap_or(if self.skip_duplicate_rows {
TableImportConflictPolicy::Skip
} else {
TableImportConflictPolicy::Error
})
}
}
fn validate_update_existing_target(
conflict_policy: TableImportConflictPolicy,
db_type: &DatabaseType,
create_table: bool,
) -> Result<(), String> {
if conflict_policy != TableImportConflictPolicy::UpdateExisting {
return Ok(());
}
if create_table {
return Err("Update-existing import requires an existing target table".to_string());
}
if !supports_primary_key_upsert(db_type) {
return Err(format!("Update-existing import conflict policy is not supported for {}", db_type.as_str()));
}
Ok(())
}
fn update_existing_primary_key_columns(
conflict_policy: TableImportConflictPolicy,
mappings: &[TableImportColumnMapping],
target_columns: &[crate::db::ColumnInfo],
) -> Result<Vec<String>, String> {
if conflict_policy != TableImportConflictPolicy::UpdateExisting {
return Ok(Vec::new());
}
let primary_key_columns = target_columns.iter().filter(|column| column.is_primary_key).collect::<Vec<_>>();
if primary_key_columns.is_empty() {
return Err("Update-existing import requires target primary-key metadata".to_string());
}
let mut mapped_primary_keys = Vec::with_capacity(primary_key_columns.len());
let mut missing_primary_keys = Vec::new();
for primary_key in primary_key_columns {
if let Some(mapping) =
mappings.iter().find(|mapping| mapping.target_column.eq_ignore_ascii_case(&primary_key.name))
{
mapped_primary_keys.push(mapping.target_column.clone());
} else {
missing_primary_keys.push(primary_key.name.clone());
}
}
if !missing_primary_keys.is_empty() {
return Err(format!(
"Update-existing import requires every primary-key column to be mapped; missing: {}",
missing_primary_keys.join(", ")
));
}
let has_mapped_non_primary_key = mappings.iter().any(|mapping| {
!mapped_primary_keys.iter().any(|primary_key| primary_key.eq_ignore_ascii_case(&mapping.target_column))
});
if !has_mapped_non_primary_key {
return Err("Update-existing import requires at least one mapped non-primary-key column".to_string());
}
Ok(mapped_primary_keys)
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct TableImportPreparedSource {
@@ -4347,7 +4430,8 @@ fn build_import_insert_batches_with_plan(
schema: &str,
db_type: &DatabaseType,
kingbase_oracle_mode: bool,
skip_duplicate_rows: bool,
conflict_policy: TableImportConflictPolicy,
primary_key_columns: &[String],
date_time_format: Option<&str>,
hard_sql_bytes: Option<usize>,
) -> Result<Vec<ImportSqlBatch>, String> {
@@ -4355,16 +4439,33 @@ fn build_import_insert_batches_with_plan(
return Ok(Vec::new());
}
let value_rows = import_value_rows_sql(rows, plan, db_type, kingbase_oracle_mode, date_time_format);
let batches = crate::data::transfer::generate_insert_typed_sql_batches_from_value_rows_with_options(
&plan.target_columns,
&value_rows,
table,
schema,
db_type,
None,
SqlBatchLimits::for_database(db_type, rows.len()).with_hard_sql_bytes(hard_sql_bytes),
skip_duplicate_rows,
)?;
let limits = SqlBatchLimits::for_database(db_type, rows.len()).with_hard_sql_bytes(hard_sql_bytes);
let batches = match conflict_policy {
TableImportConflictPolicy::Error | TableImportConflictPolicy::Skip => {
crate::data::transfer::generate_insert_typed_sql_batches_from_value_rows_with_options(
&plan.target_columns,
&value_rows,
table,
schema,
db_type,
None,
limits,
conflict_policy == TableImportConflictPolicy::Skip,
)?
}
TableImportConflictPolicy::UpdateExisting => {
crate::data::transfer::generate_primary_key_upsert_sql_batches_from_value_rows(
&plan.target_columns,
&value_rows,
table,
schema,
db_type,
None,
limits,
primary_key_columns,
)?
}
};
Ok(batches.into_iter().map(|(sql, row_count)| ImportSqlBatch { sql, row_count }).collect())
}
@@ -4429,7 +4530,8 @@ fn build_import_execution_batches(
schema: &str,
db_type: &DatabaseType,
kingbase_oracle_mode: bool,
skip_duplicate_rows: bool,
conflict_policy: TableImportConflictPolicy,
primary_key_columns: &[String],
date_time_format: Option<&str>,
hard_sql_bytes: Option<usize>,
) -> Result<Vec<ImportSqlBatch>, String> {
@@ -4441,12 +4543,13 @@ fn build_import_execution_batches(
schema,
db_type,
kingbase_oracle_mode,
skip_duplicate_rows,
conflict_policy,
primary_key_columns,
date_time_format,
hard_sql_bytes,
);
}
if *db_type == DatabaseType::CloudflareD1 {
if *db_type == DatabaseType::CloudflareD1 && conflict_policy != TableImportConflictPolicy::UpdateExisting {
return crate::db::cloudflare_d1::build_import_insert_batches(
rows,
columns,
@@ -4465,7 +4568,8 @@ fn build_import_execution_batches(
schema,
db_type,
kingbase_oracle_mode,
skip_duplicate_rows,
conflict_policy,
primary_key_columns,
date_time_format,
hard_sql_bytes,
)
@@ -4664,7 +4768,8 @@ fn build_import_insert_batches_with_format(
schema,
db_type,
kingbase_oracle_mode,
false,
TableImportConflictPolicy::Error,
&[],
date_time_format,
None,
)?);
@@ -5503,6 +5608,18 @@ fn import_batch_execution_policy(
}
}
fn completed_import_rows(
conflict_policy: TableImportConflictPolicy,
submitted_rows: usize,
affected_rows: u64,
) -> usize {
if conflict_policy == TableImportConflictPolicy::Skip {
affected_rows as usize
} else {
submitted_rows
}
}
#[allow(clippy::too_many_arguments)]
async fn execute_import_transaction(
state: &AppState,
@@ -5639,14 +5756,15 @@ async fn execute_import_rows_batch(
postgres_copy_accumulator: &mut Option<PostgresCopyAccumulator>,
sqlite_append_transaction: &mut Option<SqliteAppendTransaction>,
kingbase_oracle_mode: bool,
skip_duplicate_rows: bool,
conflict_policy: TableImportConflictPolicy,
primary_key_columns: &[String],
date_time_format: Option<&str>,
hard_sql_bytes: Option<usize>,
db_write_ms: &mut u128,
statement_count: &mut usize,
) -> Result<usize, ImportRowsBatchError> {
let execution_policy = import_batch_execution_policy(mode, pending_truncate, db_type);
if !skip_duplicate_rows {
if conflict_policy == TableImportConflictPolicy::Error {
if let Some((import_plan, bulk_plan)) = sqlserver_bulk_plans_for_rows(db_type, plan, sqlserver_bulk_plan, rows)
{
return execute_sqlserver_bulk_rows_batch(
@@ -5667,7 +5785,7 @@ async fn execute_import_rows_batch(
}
// COPY is used only for plain scalar PostgreSQL rows and ordinary tables. Any unsupported
// value or table feature falls through to the portable INSERT generator below.
if !skip_duplicate_rows
if conflict_policy == TableImportConflictPolicy::Error
&& execution_policy.allow_postgres_copy
&& *db_type == DatabaseType::Postgres
&& !rows
@@ -5711,7 +5829,8 @@ async fn execute_import_rows_batch(
schema,
db_type,
kingbase_oracle_mode,
skip_duplicate_rows,
conflict_policy,
primary_key_columns,
date_time_format,
hard_sql_bytes,
)
@@ -5774,22 +5893,19 @@ async fn execute_import_rows_batch(
)
.await
.map_err(|message| ImportRowsBatchError::with_rows_imported(rows_imported, message))?;
return Ok(if skip_duplicate_rows {
rows_imported.saturating_add(result.affected_rows as usize)
} else {
rows_imported.saturating_add(rows.len())
});
return Ok(rows_imported.saturating_add(completed_import_rows(
conflict_policy,
rows.len(),
result.affected_rows,
)));
}
for batch in batches {
ensure_import_write_allowed(import_id, is_cancelled, rows_imported).await?;
let result = execute_import_statement(state, pool_key, &batch.sql, db_write_ms, statement_count)
.await
.map_err(|error| ImportRowsBatchError::with_rows_imported(rows_imported, error))?;
rows_imported = if skip_duplicate_rows {
rows_imported.saturating_add(result.affected_rows as usize)
} else {
rows_imported.saturating_add(batch.row_count)
};
rows_imported =
rows_imported.saturating_add(completed_import_rows(conflict_policy, batch.row_count, result.affected_rows));
}
Ok(rows_imported)
}
@@ -6594,6 +6710,10 @@ where
let mut statement_count = 0usize;
let batch_size = if request.batch_size == 0 { DEFAULT_BATCH_SIZE } else { request.batch_size };
let kingbase_oracle_mode = kingbase_oracle_compatibility_mode(state, pool_key, db_type).await;
let conflict_policy = request.effective_conflict_policy();
if let Err(error) = validate_update_existing_target(conflict_policy, db_type, request.create_table) {
return Err(emit_import_error(&mut progress_callback, request, 0, 0, started_at, error));
}
let source_format = match effective_source_format(&request.file_path, request.source_format) {
Ok(format) => format,
Err(error) => {
@@ -6750,6 +6870,43 @@ where
create_table_sample = Some(parsed);
}
let target_columns = match get_columns_for_transfer(
state,
pool_key,
&request.connection_id,
&request.database,
&request.schema,
&request.table,
None,
)
.await
{
Ok(columns) => columns,
Err(error) if conflict_policy == TableImportConflictPolicy::UpdateExisting => {
return Err(emit_import_error(
&mut progress_callback,
request,
0,
0,
started_at,
format!("Update-existing import could not load target primary-key metadata: {error}"),
));
}
Err(_) => Vec::new(),
};
let primary_key_columns =
match update_existing_primary_key_columns(conflict_policy, &request.mappings, &target_columns) {
Ok(columns) => columns,
Err(error) => {
return Err(emit_import_error(&mut progress_callback, request, 0, 0, started_at, error));
}
};
let mut target_column_types =
target_columns.iter().map(|column| (column.name.clone(), column.data_type.clone())).collect::<Vec<_>>();
if target_column_types.is_empty() {
target_column_types = created_column_types.clone().unwrap_or_default();
}
if source_format.is_delimited() {
let parsed = if let Some(parsed) = create_table_sample.clone().or_else(|| prepared_source.clone()) {
parsed
@@ -6773,23 +6930,6 @@ where
let total_bytes = tokio::fs::metadata(&request.file_path).await.map(|metadata| metadata.len()).unwrap_or(0);
let mut target_column_types = get_columns_for_transfer(
state,
pool_key,
&request.connection_id,
&request.database,
&request.schema,
&request.table,
None,
)
.await
.unwrap_or_default()
.into_iter()
.map(|column| (column.name, column.data_type))
.collect::<Vec<_>>();
if target_column_types.is_empty() {
target_column_types = created_column_types.clone().unwrap_or_default();
}
let (resolved_encoding, _) =
validated_text_encoding.ok_or_else(|| "Delimited import encoding was not validated".to_string())?;
let mut streaming_options = import_parse_options.clone();
@@ -6949,7 +7089,8 @@ where
&mut postgres_copy_accumulator,
&mut sqlite_append_transaction,
kingbase_oracle_mode,
request.skip_duplicate_rows,
conflict_policy,
&primary_key_columns,
request.date_time_format.as_deref(),
import_sql_hard_limit,
&mut db_write_ms,
@@ -7141,23 +7282,6 @@ where
let effective_batch_size = effective_import_batch_size(db_type, batch_size);
let expected_columns =
create_table_sample.as_ref().or(prepared_source.as_ref()).map(|source| source.columns.clone());
let mut target_column_types = get_columns_for_transfer(
state,
pool_key,
&request.connection_id,
&request.database,
&request.schema,
&request.table,
None,
)
.await
.unwrap_or_default()
.into_iter()
.map(|column| (column.name, column.data_type))
.collect::<Vec<_>>();
if target_column_types.is_empty() {
target_column_types = created_column_types.clone().unwrap_or_default();
}
let text_source_columns = textual_source_columns_for_import(&request.mappings, &target_column_types);
// No truncate, INSERT, or COPY may run until the selected worksheet parses to EOF.
let mut last_xlsx_read_bytes = 0u64;
@@ -7452,7 +7576,8 @@ where
&mut postgres_copy_accumulator,
&mut sqlite_append_transaction,
kingbase_oracle_mode,
request.skip_duplicate_rows,
conflict_policy,
&primary_key_columns,
request.date_time_format.as_deref(),
import_sql_hard_limit,
&mut db_write_ms,
@@ -7636,23 +7761,6 @@ where
started_at,
None,
));
let mut target_column_types = get_columns_for_transfer(
state,
pool_key,
&request.connection_id,
&request.database,
&request.schema,
&request.table,
None,
)
.await
.unwrap_or_default()
.into_iter()
.map(|column| (column.name, column.data_type))
.collect::<Vec<_>>();
if target_column_types.is_empty() {
target_column_types = created_column_types.clone().unwrap_or_default();
}
let text_source_columns = textual_source_columns_for_import(&request.mappings, &target_column_types);
let effective_batch_size = effective_import_batch_size(db_type, batch_size);
let mut row_source = match ImportRowSource::open(
@@ -7801,7 +7909,8 @@ where
&mut postgres_copy_accumulator,
&mut sqlite_append_transaction,
kingbase_oracle_mode,
request.skip_duplicate_rows,
conflict_policy,
&primary_key_columns,
request.date_time_format.as_deref(),
import_sql_hard_limit,
&mut db_write_ms,
@@ -7964,6 +8073,82 @@ mod tests {
assert_eq!(progress_json["totalRowsExact"], serde_json::json!(true));
}
#[test]
fn table_import_conflict_policy_preserves_legacy_skip_requests() {
let request_json = serde_json::json!({
"importId": "import-1",
"connectionId": "connection-1",
"database": "db",
"schema": "public",
"table": "users",
"filePath": "users.csv",
"mappings": [],
"mode": "append",
"batchSize": 500
});
let default_request: TableImportRequest = serde_json::from_value(request_json.clone()).unwrap();
assert_eq!(default_request.effective_conflict_policy(), TableImportConflictPolicy::Error);
let mut legacy_skip_json = request_json.clone();
legacy_skip_json["skipDuplicateRows"] = serde_json::json!(true);
let legacy_skip: TableImportRequest = serde_json::from_value(legacy_skip_json).unwrap();
assert_eq!(legacy_skip.effective_conflict_policy(), TableImportConflictPolicy::Skip);
let mut explicit_json = request_json;
explicit_json["skipDuplicateRows"] = serde_json::json!(true);
explicit_json["conflictPolicy"] = serde_json::json!("updateExisting");
let explicit: TableImportRequest = serde_json::from_value(explicit_json).unwrap();
assert_eq!(explicit.effective_conflict_policy(), TableImportConflictPolicy::UpdateExisting);
}
#[test]
fn update_existing_progress_counts_source_rows_not_database_affected_rows() {
assert_eq!(completed_import_rows(TableImportConflictPolicy::Error, 3, 3), 3);
assert_eq!(completed_import_rows(TableImportConflictPolicy::Skip, 3, 1), 1);
// MySQL reports two affected rows for a changed row handled by ON DUPLICATE KEY UPDATE.
assert_eq!(completed_import_rows(TableImportConflictPolicy::UpdateExisting, 3, 6), 3);
}
#[test]
fn update_existing_requires_supported_dialect_and_complete_primary_key_mapping() {
let mappings = vec![
TableImportColumnMapping {
source_column: "tenant".to_string(),
target_column: "tenant_id".to_string(),
target_data_type: None,
},
TableImportColumnMapping {
source_column: "name".to_string(),
target_column: "name".to_string(),
target_data_type: None,
},
];
let target_columns = vec![
crate::db::ColumnInfo { name: "tenant_id".to_string(), is_primary_key: true, ..Default::default() },
crate::db::ColumnInfo { name: "id".to_string(), is_primary_key: true, ..Default::default() },
crate::db::ColumnInfo { name: "name".to_string(), ..Default::default() },
];
let missing_key =
update_existing_primary_key_columns(TableImportConflictPolicy::UpdateExisting, &mappings, &target_columns)
.unwrap_err();
assert!(missing_key.contains("id"));
let missing_metadata = update_existing_primary_key_columns(
TableImportConflictPolicy::UpdateExisting,
&mappings,
&[crate::db::ColumnInfo { name: "name".to_string(), ..Default::default() }],
)
.unwrap_err();
assert!(missing_metadata.contains("primary-key metadata"));
let unsupported =
validate_update_existing_target(TableImportConflictPolicy::UpdateExisting, &DatabaseType::Oracle, false)
.unwrap_err();
assert!(unsupported.contains("not supported for oracle"));
}
#[test]
fn compiled_import_plan_reuses_source_indexes_and_target_types() {
let columns = vec!["name".to_string(), "id".to_string()];
@@ -8047,6 +8232,7 @@ mod tests {
effective_encoding: Some(TableImportTextEncoding::Utf8),
}),
skip_duplicate_rows: false,
conflict_policy: None,
retain_source: false,
};
@@ -9761,6 +9947,7 @@ mod tests {
date_time_format: None,
prepared_source: None,
skip_duplicate_rows: false,
conflict_policy: None,
retain_source: false,
};
let started_at = Instant::now();
@@ -10172,6 +10359,7 @@ mod tests {
date_time_format: None,
prepared_source: None,
skip_duplicate_rows: false,
conflict_policy: None,
retain_source: false,
};
@@ -10262,6 +10450,7 @@ mod tests {
date_time_format: None,
prepared_source: None,
skip_duplicate_rows: false,
conflict_policy: None,
retain_source: false,
};
@@ -10343,6 +10532,7 @@ mod tests {
date_time_format: None,
prepared_source: None,
skip_duplicate_rows: false,
conflict_policy: None,
retain_source: false,
};
@@ -11426,6 +11616,99 @@ mod tests {
]);
}
#[test]
fn import_conflict_policy_keeps_default_and_skip_sql_unchanged() {
let plan = CompiledImportPlan {
mapped_source_indexes: vec![0, 1],
target_columns: vec!["id".to_string(), "name".to_string()],
column_types: vec![Some("integer".to_string()), Some("text".to_string())],
};
let rows = vec![vec![serde_json::json!(1), serde_json::json!("Ada")]];
let default_batches = build_import_insert_batches_with_plan(
&rows,
&plan,
"users",
"public",
&DatabaseType::Postgres,
false,
TableImportConflictPolicy::Error,
&[],
None,
None,
)
.unwrap();
assert_eq!(default_batches[0].sql, "INSERT INTO \"public\".\"users\" (\"id\", \"name\") VALUES\n(1, 'Ada')");
let skip_batches = build_import_insert_batches_with_plan(
&rows,
&plan,
"users",
"public",
&DatabaseType::Postgres,
false,
TableImportConflictPolicy::Skip,
&[],
None,
None,
)
.unwrap();
assert_eq!(
skip_batches[0].sql,
"INSERT INTO \"public\".\"users\" (\"id\", \"name\") VALUES\n(1, 'Ada')\nON CONFLICT DO NOTHING"
);
}
#[test]
fn update_existing_sql_uses_composite_primary_key_and_only_updates_non_keys() {
let plan = CompiledImportPlan {
mapped_source_indexes: vec![0, 1, 2, 3],
target_columns: vec!["tenant_id".to_string(), "id".to_string(), "name".to_string(), "status".to_string()],
column_types: vec![None, None, None, None],
};
let rows = vec![vec![
serde_json::json!(7),
serde_json::json!(42),
serde_json::json!("Ada"),
serde_json::json!("active"),
]];
let postgres = build_import_insert_batches_with_plan(
&rows,
&plan,
"users",
"public",
&DatabaseType::Postgres,
false,
TableImportConflictPolicy::UpdateExisting,
&["tenant_id".to_string(), "id".to_string()],
None,
None,
)
.unwrap();
assert!(postgres[0].sql.contains("ON CONFLICT (\"tenant_id\", \"id\") DO UPDATE SET"));
assert!(postgres[0].sql.contains("\"name\" = EXCLUDED.\"name\""));
assert!(postgres[0].sql.contains("\"status\" = EXCLUDED.\"status\""));
assert!(!postgres[0].sql.contains("\"id\" = EXCLUDED.\"id\""));
assert_eq!(postgres[0].row_count, 1);
let mysql = build_import_insert_batches_with_plan(
&rows,
&plan,
"users",
"",
&DatabaseType::Mysql,
false,
TableImportConflictPolicy::UpdateExisting,
&["tenant_id".to_string(), "id".to_string()],
None,
None,
)
.unwrap();
assert!(mysql[0].sql.contains("ON DUPLICATE KEY UPDATE `name` = VALUES(`name`), `status` = VALUES(`status`)"));
assert!(!mysql[0].sql.contains("`id` = VALUES(`id`)"));
}
#[test]
fn iris_import_uses_single_row_values_statements() {
let mappings = vec![TableImportColumnMapping {
@@ -11698,7 +11981,8 @@ mod tests {
"dbo",
&DatabaseType::SqlServer,
false,
false,
TableImportConflictPolicy::Error,
&[],
None,
None,
)
@@ -12024,7 +12308,8 @@ mod tests {
&mut sqlite_append_transaction,
&mut postgres_copy_accumulator,
false,
false,
TableImportConflictPolicy::Error,
&[],
None,
None,
&mut db_write_ms,
@@ -12242,7 +12527,8 @@ mod tests {
&mut self.postgres_copy_accumulator,
&mut self.transaction,
false,
false,
TableImportConflictPolicy::Error,
&[],
None,
None,
&mut self.db_write_ms,
@@ -12368,6 +12654,7 @@ mod tests {
date_time_format: None,
prepared_source: None,
skip_duplicate_rows: false,
conflict_policy: None,
retain_source: false,
};
@@ -12395,6 +12682,97 @@ mod tests {
);
}
#[tokio::test]
async fn sqlite_update_existing_import_counts_updated_rows_once() {
let dir = tempfile::tempdir().unwrap();
let storage = crate::persistence::test_storage::open(&dir.path().join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let connection_id = "sqlite-update-existing";
let pool_key = format!("{connection_id}:session:import");
let database_path = dir.path().join("target.db");
let sqlite = crate::db::sqlite::connect_path_create_if_missing(database_path.to_str().unwrap()).await.unwrap();
crate::db::sqlite::execute_query(
&sqlite,
"CREATE TABLE items (tenant_id INTEGER, id INTEGER, name TEXT, PRIMARY KEY (tenant_id, id))",
)
.await
.unwrap();
crate::db::sqlite::execute_query(&sqlite, "INSERT INTO items (tenant_id, id, name) VALUES (1, 1, 'Old')")
.await
.unwrap();
state
.update_connection_pools(|connections| {
connections.insert(pool_key.clone(), PoolKind::Sqlite(sqlite.clone()));
})
.await;
let config: ConnectionConfig = serde_json::from_value(serde_json::json!({
"id": connection_id,
"name": "SQLite update-existing test",
"db_type": "sqlite",
"host": "",
"port": 0,
"username": "",
"password": "",
"database": database_path.to_string_lossy()
}))
.unwrap();
state.configs.write().await.insert(connection_id.to_string(), config);
let data_path = dir.path().join("rows.csv");
std::fs::write(&data_path, b"tenant_id,id,name\n1,1,Updated\n1,2,Inserted\n").unwrap();
let request = TableImportRequest {
import_id: "sqlite-update-existing".to_string(),
connection_id: connection_id.to_string(),
database: String::new(),
schema: String::new(),
table: "items".to_string(),
file_path: data_path.to_string_lossy().to_string(),
source_ref: None,
source_format: Some(TableImportSourceFormat::Csv),
parse_options: TableImportParseOptions::default(),
mappings: ["tenant_id", "id", "name"]
.into_iter()
.map(|column| TableImportColumnMapping {
source_column: column.to_string(),
target_column: column.to_string(),
target_data_type: None,
})
.collect(),
mode: TableImportMode::Append,
create_table: false,
batch_size: 2,
date_time_format: None,
prepared_source: None,
retain_source: false,
conflict_policy: Some(TableImportConflictPolicy::UpdateExisting),
skip_duplicate_rows: false,
};
let summary = import_table_file_core(
&state,
&request,
&DatabaseType::Sqlite,
&pool_key,
|_| Box::pin(async { false }),
|_| {},
)
.await
.unwrap();
assert_eq!(summary.rows_imported, 2);
let rows =
crate::db::sqlite::execute_query(&sqlite, "SELECT tenant_id, id, name FROM items ORDER BY tenant_id, id")
.await
.unwrap()
.rows;
assert_eq!(
rows,
vec![
vec![serde_json::json!(1), serde_json::json!(1), serde_json::json!("Updated")],
vec![serde_json::json!(1), serde_json::json!(2), serde_json::json!("Inserted")],
]
);
}
#[tokio::test]
async fn preview_missing_source_fails_before_parsing() {
let path = std::env::temp_dir().join(format!("dbx-missing-import-{}.csv", uuid::Uuid::new_v4()));
+119
View File
@@ -4036,6 +4036,76 @@ fn uses_mysql_style_upsert(db_type: &DatabaseType) -> bool {
matches!(db_type, DatabaseType::Mysql | DatabaseType::Doris | DatabaseType::StarRocks | DatabaseType::OpenGauss)
}
pub(crate) fn supports_primary_key_upsert(db_type: &DatabaseType) -> bool {
matches!(
db_type,
DatabaseType::Postgres
| DatabaseType::Kingbase
| DatabaseType::Sqlite
| DatabaseType::CloudflareD1
| DatabaseType::DuckDb
| DatabaseType::Mysql
| DatabaseType::Doris
| DatabaseType::StarRocks
| DatabaseType::OpenGauss
)
}
fn primary_key_upsert_clause(
columns: &[String],
pk_columns: &[String],
db_type: &DatabaseType,
quote_target_column_names: bool,
) -> Result<String, String> {
if pk_columns.is_empty() {
return Err("Update-existing import requires target primary-key metadata".to_string());
}
if pk_columns.iter().any(|primary_key| !columns.iter().any(|column| column.eq_ignore_ascii_case(primary_key))) {
return Err("Update-existing import requires every target primary-key column to be mapped".to_string());
}
let non_pk_columns = columns
.iter()
.filter(|column| !pk_columns.iter().any(|primary_key| column.eq_ignore_ascii_case(primary_key)))
.collect::<Vec<_>>();
if non_pk_columns.is_empty() {
return Err("Update-existing import requires at least one mapped non-primary-key column".to_string());
}
if (is_postgres_transfer_dialect(db_type) && !matches!(db_type, DatabaseType::OpenGauss))
|| matches!(db_type, DatabaseType::Sqlite | DatabaseType::CloudflareD1 | DatabaseType::DuckDb)
{
let primary_keys = pk_columns
.iter()
.map(|column| transfer_column_identifier(column, db_type, quote_target_column_names))
.collect::<Vec<_>>()
.join(", ");
let updates = non_pk_columns
.iter()
.map(|column| {
let column = transfer_column_identifier(column, db_type, quote_target_column_names);
format!("{column} = EXCLUDED.{column}")
})
.collect::<Vec<_>>()
.join(", ");
return Ok(format!("\nON CONFLICT ({primary_keys}) DO UPDATE SET {updates}"));
}
if uses_mysql_style_upsert(db_type) {
let updates = non_pk_columns
.iter()
.map(|column| {
let column = transfer_column_identifier(column, db_type, quote_target_column_names);
format!("{column} = VALUES({column})")
})
.collect::<Vec<_>>()
.join(", ");
return Ok(format!("\nON DUPLICATE KEY UPDATE {updates}"));
}
Err(format!("Update-existing import conflict policy is not supported for {}", db_type.as_str()))
}
#[allow(clippy::too_many_arguments)]
fn generate_upsert_typed_for_transfer(
columns: &[String],
@@ -4747,6 +4817,55 @@ pub(crate) fn generate_insert_typed_sql_batches_from_value_rows_with_options(
Ok(statements)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn generate_primary_key_upsert_sql_batches_from_value_rows(
columns: &[String],
value_rows: &[String],
table: &str,
schema: &str,
db_type: &DatabaseType,
catalog: Option<&str>,
limits: SqlBatchLimits,
pk_columns: &[String],
) -> Result<Vec<(String, usize)>, String> {
if !supports_primary_key_upsert(db_type) {
return Err(format!("Update-existing import conflict policy is not supported for {}", db_type.as_str()));
}
let clause = primary_key_upsert_clause(columns, pk_columns, db_type, true)?;
let clause_bytes = sql_text_bytes(&clause, db_type);
let adjusted_limits = SqlBatchLimits {
max_rows: limits.max_rows,
target_sql_bytes: limits.target_sql_bytes.saturating_sub(clause_bytes).max(1),
hard_sql_bytes: limits.hard_sql_bytes.map(|limit| limit.saturating_sub(clause_bytes).max(1)),
};
let batches = generate_insert_sql_batches_from_value_rows(
columns,
value_rows,
table,
schema,
db_type,
catalog,
adjusted_limits,
false,
true,
)?;
batches
.into_iter()
.map(|(mut sql, row_count)| {
sql.push_str(&clause);
if limits.hard_sql_bytes.is_some_and(|limit| sql_text_bytes(&sql, db_type) > limit) {
return Err(format!(
"SQL batch with update-existing conflict handling exceeds the {} byte hard limit",
limits.hard_sql_bytes.unwrap_or_default()
));
}
Ok((sql, row_count))
})
.collect()
}
#[allow(clippy::too_many_arguments)]
fn generate_insert_typed_sql_batches_for_transfer(
columns: &[String],
@@ -133,6 +133,7 @@ fn live_sqlserver_import_request(
date_time_format: None,
prepared_source: None,
retain_source: false,
conflict_policy: None,
skip_duplicate_rows: false,
}
}