1 parent
e9998e2806
commit
0a6ab8d360
28 files changed
+923
-85
No files matched your search
@@ -22,7 +22,7 @@ use crate::transfer::{
|
||||
keyset_pagination_sql_with_identifier_quote, quote_identifier, quote_postgres_string_literal,
|
||||
wrap_dameng_identity_insert_sql_for_table,
|
||||
};
|
||||
use crate::types::{ObjectSourceKind, SpatialColumn};
|
||||
use crate::types::{ObjectSourceKind, SpatialColumn, SqlExportColumnSelection};
|
||||
|
||||
static EXPORT_CANCELLED: std::sync::LazyLock<RwLock<HashSet<String>>> =
|
||||
std::sync::LazyLock::new(|| RwLock::new(HashSet::new()));
|
||||
@@ -1444,6 +1444,164 @@ pub(crate) fn is_internal_export_column(database_type: Option<DatabaseType>, col
|
||||
&& column.eq_ignore_ascii_case(crate::sql_dialect::DBX_ROWID_COLUMN)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct SqlExportProjection {
|
||||
entries: Vec<(usize, usize)>,
|
||||
}
|
||||
|
||||
impl SqlExportProjection {
|
||||
pub(crate) fn resolve(
|
||||
columns: &[String],
|
||||
selected_columns: Option<&[SqlExportColumnSelection]>,
|
||||
) -> Result<Self, String> {
|
||||
let Some(selected_columns) = selected_columns else {
|
||||
return Ok(Self { entries: (0..columns.len()).map(|index| (index, index)).collect() });
|
||||
};
|
||||
if selected_columns.is_empty() {
|
||||
return Err("Select at least one column for SQL export.".to_string());
|
||||
}
|
||||
|
||||
let mut entries = Vec::with_capacity(selected_columns.len());
|
||||
let mut used = HashSet::with_capacity(selected_columns.len());
|
||||
let mut indexes_by_name = HashMap::<&str, Vec<usize>>::new();
|
||||
for (index, name) in columns.iter().enumerate() {
|
||||
indexes_by_name.entry(name.as_str()).or_default().push(index);
|
||||
}
|
||||
for selected in selected_columns {
|
||||
let current_index = indexes_by_name
|
||||
.get(selected.name.as_str())
|
||||
.and_then(|indexes| indexes.get(selected.name_occurrence))
|
||||
.copied()
|
||||
.ok_or_else(|| format!("Selected SQL export column was not found: {}", selected.name))?;
|
||||
if !used.insert(current_index) {
|
||||
return Err(format!("SQL export column was selected more than once: {}", selected.name));
|
||||
}
|
||||
entries.push((current_index, selected.source_index));
|
||||
}
|
||||
Ok(Self { entries })
|
||||
}
|
||||
|
||||
pub(crate) fn project<T: Clone + Default>(&self, values: &[T]) -> Vec<T> {
|
||||
self.entries.iter().map(|(current_index, _)| values.get(*current_index).cloned().unwrap_or_default()).collect()
|
||||
}
|
||||
|
||||
pub(crate) fn project_request<T: Clone + Default>(&self, values: &[T]) -> Vec<T> {
|
||||
self.entries.iter().map(|(_, request_index)| values.get(*request_index).cloned().unwrap_or_default()).collect()
|
||||
}
|
||||
|
||||
pub(crate) fn project_owned<T: Default>(&self, mut values: Vec<T>) -> Vec<T> {
|
||||
self.entries.iter().map(|(index, _)| values.get_mut(*index).map(std::mem::take).unwrap_or_default()).collect()
|
||||
}
|
||||
|
||||
pub(crate) fn project_spatial_columns(&self, columns: &[SpatialColumn]) -> Vec<SpatialColumn> {
|
||||
let spatial_by_index: HashMap<_, _> = columns.iter().map(|column| (column.column_index, column)).collect();
|
||||
self.entries
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(projected_index, (current_index, _))| {
|
||||
spatial_by_index
|
||||
.get(current_index)
|
||||
.map(|column| SpatialColumn { column_index: projected_index, srid: column.srid })
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn project_insert_options(
|
||||
&self,
|
||||
mut options: BuildExportInsertStatementsOptions,
|
||||
) -> BuildExportInsertStatementsOptions {
|
||||
options.columns = self.project_owned(options.columns);
|
||||
options.column_types = self.project_owned(options.column_types);
|
||||
options.column_extras = self.project_owned(options.column_extras);
|
||||
options.spatial_columns = self.project_spatial_columns(&options.spatial_columns);
|
||||
options.spatial_values = options.spatial_values.into_iter().map(|row| self.project_owned(row)).collect();
|
||||
options.rows = options.rows.into_iter().map(|row| self.project_owned(row)).collect();
|
||||
options
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod sql_export_projection_tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn selection(index: usize, name: &str, occurrence: usize) -> SqlExportColumnSelection {
|
||||
SqlExportColumnSelection { source_index: index, name: name.into(), name_occurrence: occurrence }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn omitted_selection_keeps_every_column_but_empty_selection_is_rejected() {
|
||||
let columns = vec!["id".into(), "name".into()];
|
||||
let projection = SqlExportProjection::resolve(&columns, None).unwrap();
|
||||
assert_eq!(projection.project(&columns), columns);
|
||||
assert!(SqlExportProjection::resolve(&columns, Some(&[])).unwrap_err().contains("at least one"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicate_names_follow_occurrence_after_reordering() {
|
||||
let columns = vec!["id".into(), "id".into(), "name".into()];
|
||||
let projection =
|
||||
SqlExportProjection::resolve(&columns, Some(&[selection(3, "id", 1), selection(2, "name", 0)])).unwrap();
|
||||
assert_eq!(projection.project(&[json!(1), json!(2), json!("Ada")]), vec![json!(2), json!("Ada")]);
|
||||
assert_eq!(projection.project_request(&[0, 1, 2, 3]), vec![3, 2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_and_repeated_column_identities_are_rejected() {
|
||||
let columns = vec!["id".into()];
|
||||
assert!(SqlExportProjection::resolve(&columns, Some(&[selection(0, "id", 1)]))
|
||||
.unwrap_err()
|
||||
.contains("not found"));
|
||||
assert!(SqlExportProjection::resolve(&columns, Some(&[selection(0, "name", 0)]))
|
||||
.unwrap_err()
|
||||
.contains("not found"));
|
||||
assert!(SqlExportProjection::resolve(&columns, Some(&[selection(0, "id", 0), selection(1, "id", 0)]))
|
||||
.unwrap_err()
|
||||
.contains("more than once"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn insert_projection_aligns_types_extras_spatial_data_and_values() {
|
||||
let columns = vec!["id".into(), "shape".into(), "blob".into(), "optional".into()];
|
||||
let projection = SqlExportProjection::resolve(
|
||||
&columns,
|
||||
Some(&[selection(2, "blob", 0), selection(1, "shape", 0), selection(3, "optional", 0)]),
|
||||
)
|
||||
.unwrap();
|
||||
let options = BuildExportInsertStatementsOptions {
|
||||
database_type: Some(DatabaseType::Mysql),
|
||||
identifier_quote: None,
|
||||
schema: None,
|
||||
table_name: Some("items".into()),
|
||||
qualified_table_name: None,
|
||||
columns: columns.clone(),
|
||||
column_types: vec![Some("int".into()), Some("geometry".into()), Some("blob".into()), Some("text".into())],
|
||||
column_extras: vec![Some("auto_increment".into()), None, None, None],
|
||||
spatial_columns: vec![SpatialColumn { column_index: 1, srid: Some(4326) }],
|
||||
spatial_values: vec![vec![None, Some(4326), None, None]],
|
||||
rows: vec![vec![json!(1), json!("POINT(1 2)"), json!([0, 255]), Value::Null]],
|
||||
batch_size: Some(100),
|
||||
};
|
||||
let projected = projection.project_insert_options(options);
|
||||
assert_eq!(columns, vec!["id", "shape", "blob", "optional"]);
|
||||
assert_eq!(projected.columns, vec!["blob", "shape", "optional"]);
|
||||
assert_eq!(projected.column_types, vec![Some("blob".into()), Some("geometry".into()), Some("text".into())]);
|
||||
assert_eq!(projected.column_extras, vec![None, None, None]);
|
||||
assert_eq!(projected.spatial_columns[0].column_index, 1);
|
||||
assert_eq!(projected.spatial_values, vec![vec![None, Some(4326), None]]);
|
||||
assert_eq!(projected.rows, vec![vec![json!([0, 255]), json!("POINT(1 2)"), Value::Null]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wide_duplicate_selection_preserves_every_position() {
|
||||
let columns = vec!["value".to_string(); 10000];
|
||||
let selected: Vec<_> = (0..columns.len()).map(|index| selection(index, "value", index)).collect();
|
||||
let projection = SqlExportProjection::resolve(&columns, Some(&selected)).unwrap();
|
||||
let values: Vec<_> = (0..columns.len()).collect();
|
||||
assert_eq!(projection.project_owned(values.clone()), values);
|
||||
}
|
||||
}
|
||||
|
||||
fn is_postgres_tsvector_export_column(database_type: Option<DatabaseType>, column_type: Option<&str>) -> bool {
|
||||
database_type == Some(DatabaseType::Postgres)
|
||||
&& column_type
|
||||
|
||||
@@ -16,7 +16,8 @@ use crate::csv_export::{
|
||||
};
|
||||
pub use crate::database_export::ExportStatus;
|
||||
use crate::database_export::{
|
||||
build_export_insert_statements_excluding, is_export_cancelled, BuildExportInsertStatementsOptions, SqlInsertMode,
|
||||
build_export_insert_statements_excluding, is_export_cancelled, BuildExportInsertStatementsOptions,
|
||||
SqlExportProjection, SqlInsertMode,
|
||||
};
|
||||
use crate::models::connection::DatabaseType;
|
||||
use crate::query::{
|
||||
@@ -112,6 +113,8 @@ pub struct QueryResultExportRequest {
|
||||
/// Frontend sends these in original full-query column order.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub export_column_types: Option<Vec<Option<String>>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selected_columns: Option<Vec<crate::types::SqlExportColumnSelection>>,
|
||||
/// Column EXTRA metadata (for example SQL Server/Dameng `identity`) for SQL
|
||||
/// INSERT export, sent by the frontend from the result's table metadata.
|
||||
/// Without it the exported INSERT writes identity values without
|
||||
@@ -296,19 +299,29 @@ fn stream_export_was_cancelled(error: &str, token_cancelled: bool, export_cancel
|
||||
/// Missing, `None`, or empty overrides infer only MySQL spatial types, which
|
||||
/// preserves the historical literal formatting of other database types.
|
||||
fn sql_insert_column_types(request: &QueryResultExportRequest, column_types: &[String]) -> Vec<Option<String>> {
|
||||
resolve_sql_insert_column_types(
|
||||
request.database_type,
|
||||
column_types,
|
||||
request.export_column_types.as_deref().unwrap_or_default(),
|
||||
)
|
||||
}
|
||||
|
||||
fn resolve_sql_insert_column_types(
|
||||
database_type: DatabaseType,
|
||||
column_types: &[String],
|
||||
overrides: &[Option<String>],
|
||||
) -> Vec<Option<String>> {
|
||||
column_types
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, inferred)| {
|
||||
request
|
||||
.export_column_types
|
||||
.as_ref()
|
||||
.and_then(|overrides| overrides.get(index))
|
||||
overrides
|
||||
.get(index)
|
||||
.and_then(|override_type| override_type.as_deref())
|
||||
.filter(|override_type| !override_type.is_empty())
|
||||
.map(str::to_string)
|
||||
.or_else(|| {
|
||||
(request.database_type == DatabaseType::Mysql
|
||||
(database_type == DatabaseType::Mysql
|
||||
&& crate::database_export::is_mysql_spatial_export_type(inferred))
|
||||
.then(|| inferred.clone())
|
||||
})
|
||||
@@ -337,6 +350,7 @@ struct SqlInsertWriter {
|
||||
schema: Option<String>,
|
||||
table_name: String,
|
||||
identifier_quote: Option<String>,
|
||||
projection: Option<SqlExportProjection>,
|
||||
/// 生成 INSERT 时需要排除的列名,来自请求里的“不含主键”设置。
|
||||
exclude_columns: Vec<String>,
|
||||
}
|
||||
@@ -426,6 +440,7 @@ impl SqlInsertWriter {
|
||||
schema: request.schema.clone(),
|
||||
table_name,
|
||||
identifier_quote: request.identifier_quote.clone(),
|
||||
projection: None,
|
||||
exclude_columns: if request.exclude_primary_keys { request.primary_keys.clone() } else { Vec::new() },
|
||||
})
|
||||
}
|
||||
@@ -439,18 +454,42 @@ impl SqlInsertWriter {
|
||||
result_column_types: &[String],
|
||||
spatial_columns: &[SpatialColumn],
|
||||
request: &QueryResultExportRequest,
|
||||
) {
|
||||
self.column_types = sql_insert_column_types(request, result_column_types);
|
||||
) -> Result<(), String> {
|
||||
let projection = SqlExportProjection::resolve(&columns, request.selected_columns.as_deref())?;
|
||||
self.column_types = if request.selected_columns.is_some() {
|
||||
resolve_sql_insert_column_types(
|
||||
request.database_type,
|
||||
&projection.project(result_column_types),
|
||||
&projection.project_request(request.export_column_types.as_deref().unwrap_or_default()),
|
||||
)
|
||||
} else {
|
||||
sql_insert_column_types(request, result_column_types)
|
||||
};
|
||||
// Column extras come from the request and align with the result columns the
|
||||
// same way `export_column_types` does; missing entries simply mean "unknown".
|
||||
self.column_extras = request.export_column_extras.clone().unwrap_or_default();
|
||||
self.spatial_columns = spatial_columns.to_vec();
|
||||
self.columns = columns;
|
||||
self.column_extras = projection.project_request(request.export_column_extras.as_deref().unwrap_or_default());
|
||||
self.spatial_columns = projection.project_spatial_columns(spatial_columns);
|
||||
self.columns = projection.project(&columns);
|
||||
self.projection = request.selected_columns.as_ref().map(|_| projection);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_row(&mut self, row: Vec<Value>, spatial_values: Option<Vec<Option<u32>>>) -> Result<(), String> {
|
||||
let row = match &self.projection {
|
||||
Some(projection) => projection.project_owned(row),
|
||||
None => row,
|
||||
};
|
||||
let spatial_values = spatial_values.unwrap_or_default();
|
||||
let spatial_values = if spatial_values.is_empty() {
|
||||
spatial_values
|
||||
} else {
|
||||
match &self.projection {
|
||||
Some(projection) => projection.project_owned(spatial_values),
|
||||
None => spatial_values,
|
||||
}
|
||||
};
|
||||
self.pending_rows.push(row);
|
||||
self.pending_spatial_values.push(spatial_values.unwrap_or_default());
|
||||
self.pending_spatial_values.push(spatial_values);
|
||||
if self.insert_mode.flush_each_row() || self.pending_rows.len() >= SQL_INSERT_BATCH_SIZE {
|
||||
self.flush_batch()?;
|
||||
}
|
||||
@@ -647,7 +686,20 @@ fn single_execution_row_bound(request: &QueryResultExportRequest) -> Option<usiz
|
||||
/// page payload, so later pages may reorder, add, or drop columns; values must
|
||||
/// follow their column name instead of their position in the page.
|
||||
fn export_column_remap(page_columns: &[String], header_columns: &[String]) -> Vec<Option<usize>> {
|
||||
header_columns.iter().map(|column| page_columns.iter().position(|page_column| page_column == column)).collect()
|
||||
let mut occurrences = std::collections::HashMap::<&str, usize>::new();
|
||||
let mut indexes_by_name = std::collections::HashMap::<&str, Vec<usize>>::new();
|
||||
for (index, name) in page_columns.iter().enumerate() {
|
||||
indexes_by_name.entry(name.as_str()).or_default().push(index);
|
||||
}
|
||||
header_columns
|
||||
.iter()
|
||||
.map(|column| {
|
||||
let occurrence = occurrences.entry(column.as_str()).or_default();
|
||||
let index = indexes_by_name.get(column.as_str()).and_then(|indexes| indexes.get(*occurrence)).copied();
|
||||
*occurrence += 1;
|
||||
index
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Reorders one page row (or a page-level cell list such as column types) into
|
||||
@@ -1010,7 +1062,7 @@ async fn export_query_result_core_inner(
|
||||
columns = result.columns.clone();
|
||||
column_types = result.column_types.clone();
|
||||
if let Some(writer) = sql_writer.as_mut() {
|
||||
writer.set_columns(columns.clone(), &column_types, &result.spatial_columns, request);
|
||||
writer.set_columns(columns.clone(), &column_types, &result.spatial_columns, request)?;
|
||||
}
|
||||
column_types.clone()
|
||||
} else if result.columns == columns {
|
||||
@@ -1247,7 +1299,7 @@ async fn try_export_postgres_query_result_stream(
|
||||
columns = stream_columns;
|
||||
temporal_column_types = column_types.clone();
|
||||
if let Some(writer) = sql_writer.as_mut() {
|
||||
writer.set_columns(columns.clone(), &column_types, &[], request);
|
||||
writer.set_columns(columns.clone(), &column_types, &[], request)?;
|
||||
} else if let Some(file) = text_file.as_mut() {
|
||||
let header = format_text_export_header(format, &columns, request.csv_quote_mode);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||
@@ -1493,7 +1545,7 @@ async fn try_export_mysql_query_result_stream(
|
||||
columns = stream_columns;
|
||||
temporal_column_types = column_types.clone();
|
||||
if let Some(writer) = sql_writer.as_mut() {
|
||||
writer.set_columns(columns.clone(), &column_types, &[], request);
|
||||
writer.set_columns(columns.clone(), &column_types, &[], request)?;
|
||||
} else if let Some(file) = text_file.as_mut() {
|
||||
let header = format_text_export_header(format, &columns, request.csv_quote_mode);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||
@@ -1718,7 +1770,7 @@ async fn try_export_clickhouse_query_result_stream(
|
||||
columns = stream_columns;
|
||||
temporal_column_types = column_types.clone();
|
||||
if let Some(writer) = sql_writer.as_mut() {
|
||||
writer.set_columns(columns.clone(), &column_types, &[], request);
|
||||
writer.set_columns(columns.clone(), &column_types, &[], request)?;
|
||||
} else if let Some(file) = text_file.as_mut() {
|
||||
let header = format_text_export_header(format, &columns, request.csv_quote_mode);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||
@@ -1912,7 +1964,7 @@ async fn try_export_sqlserver_query_result_stream(
|
||||
columns = stream_columns.to_vec();
|
||||
temporal_column_types = column_types.to_vec();
|
||||
if let Some(writer) = sql_writer.as_mut() {
|
||||
writer.set_columns(columns.clone(), &temporal_column_types, &[], request);
|
||||
writer.set_columns(columns.clone(), &temporal_column_types, &[], request)?;
|
||||
} else if let Some(file) = text_file.as_mut() {
|
||||
let header = format_text_export_header(format, &columns, request.csv_quote_mode);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||
@@ -2131,6 +2183,7 @@ mod tests {
|
||||
date_time_format: None,
|
||||
export_table_name: Some("gen_table".to_string()),
|
||||
export_column_types: Some(vec![Some("int".to_string()), Some("nvarchar(200)".to_string())]),
|
||||
selected_columns: None,
|
||||
export_column_extras,
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
@@ -2151,12 +2204,14 @@ mod tests {
|
||||
sqlserver_identity_export_request(&file_path, Some(vec![Some("identity(1,1)".to_string()), None]));
|
||||
|
||||
let mut writer = SqlInsertWriter::create(&request).expect("create sql insert writer");
|
||||
writer.set_columns(
|
||||
vec!["table_id".to_string(), "table_name".to_string()],
|
||||
&["int".to_string(), "nvarchar(200)".to_string()],
|
||||
&[],
|
||||
&request,
|
||||
);
|
||||
writer
|
||||
.set_columns(
|
||||
vec!["table_id".to_string(), "table_name".to_string()],
|
||||
&["int".to_string(), "nvarchar(200)".to_string()],
|
||||
&[],
|
||||
&request,
|
||||
)
|
||||
.expect("set SQL export columns");
|
||||
writer.write_row(vec![json!(23), json!("t_destype")], None).expect("write export row");
|
||||
writer.finish().expect("finish export");
|
||||
|
||||
@@ -2173,12 +2228,14 @@ mod tests {
|
||||
let request = sqlserver_identity_export_request(&file_path, None);
|
||||
|
||||
let mut writer = SqlInsertWriter::create(&request).expect("create sql insert writer");
|
||||
writer.set_columns(
|
||||
vec!["table_id".to_string(), "table_name".to_string()],
|
||||
&["int".to_string(), "nvarchar(200)".to_string()],
|
||||
&[],
|
||||
&request,
|
||||
);
|
||||
writer
|
||||
.set_columns(
|
||||
vec!["table_id".to_string(), "table_name".to_string()],
|
||||
&["int".to_string(), "nvarchar(200)".to_string()],
|
||||
&[],
|
||||
&request,
|
||||
)
|
||||
.expect("set SQL export columns");
|
||||
writer.write_row(vec![json!(23), json!("t_destype")], None).expect("write export row");
|
||||
writer.finish().expect("finish export");
|
||||
|
||||
@@ -2216,6 +2273,7 @@ mod tests {
|
||||
date_time_format: None,
|
||||
export_table_name: Some("users".to_string()),
|
||||
export_column_types: None,
|
||||
selected_columns: None,
|
||||
export_column_extras: None,
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
@@ -2228,12 +2286,14 @@ mod tests {
|
||||
};
|
||||
|
||||
let mut writer = SqlInsertWriter::create(&request).expect("create sql insert writer");
|
||||
writer.set_columns(
|
||||
vec!["id".to_string(), "name".to_string()],
|
||||
&["integer".to_string(), "text".to_string()],
|
||||
&[],
|
||||
&request,
|
||||
);
|
||||
writer
|
||||
.set_columns(
|
||||
vec!["id".to_string(), "name".to_string()],
|
||||
&["integer".to_string(), "text".to_string()],
|
||||
&[],
|
||||
&request,
|
||||
)
|
||||
.expect("set SQL export columns");
|
||||
writer.write_row(vec![json!(1), json!("Ada")], None).expect("write export row");
|
||||
writer.finish().expect("finish export");
|
||||
|
||||
@@ -2299,6 +2359,7 @@ mod tests {
|
||||
csv_quote_mode: CsvQuoteMode::All,
|
||||
export_table_name: None,
|
||||
export_column_types: None,
|
||||
selected_columns: None,
|
||||
export_column_extras: None,
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
@@ -2319,12 +2380,14 @@ mod tests {
|
||||
req.insert_mode = mode;
|
||||
|
||||
let mut writer = SqlInsertWriter::create(&req).expect("create SQL writer");
|
||||
writer.set_columns(
|
||||
vec!["id".to_string(), "name".to_string()],
|
||||
&["int".to_string(), "text".to_string()],
|
||||
&[],
|
||||
&req,
|
||||
);
|
||||
writer
|
||||
.set_columns(
|
||||
vec!["id".to_string(), "name".to_string()],
|
||||
&["int".to_string(), "text".to_string()],
|
||||
&[],
|
||||
&req,
|
||||
)
|
||||
.expect("set SQL export columns");
|
||||
writer.write_row(vec![serde_json::json!(1), serde_json::json!("Ada")], None).expect("write first row");
|
||||
writer.write_row(vec![serde_json::json!(2), serde_json::json!("Lin")], None).expect("write second row");
|
||||
writer.finish().expect("finish SQL writer");
|
||||
@@ -2341,6 +2404,106 @@ mod tests {
|
||||
assert_eq!(decoded.insert_mode, SqlInsertMode::Batch);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_export_column_selection_is_optional_and_round_trips() {
|
||||
let mut serialized = serde_json::to_value(request("sql", None, None)).unwrap();
|
||||
serialized.as_object_mut().unwrap().remove("selectedColumns");
|
||||
let mut decoded: QueryResultExportRequest = serde_json::from_value(serialized).unwrap();
|
||||
assert!(decoded.selected_columns.is_none());
|
||||
decoded.selected_columns = Some(vec![crate::types::SqlExportColumnSelection {
|
||||
source_index: 2,
|
||||
name: "id".into(),
|
||||
name_occurrence: 1,
|
||||
}]);
|
||||
let encoded = serde_json::to_value(&decoded).unwrap();
|
||||
assert_eq!(encoded["selectedColumns"], json!([{ "sourceIndex": 2, "name": "id", "nameOccurrence": 1 }]));
|
||||
let round_trip: QueryResultExportRequest = serde_json::from_value(encoded).unwrap();
|
||||
assert_eq!(round_trip.selected_columns.unwrap()[0].name_occurrence, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_insert_writer_projects_duplicate_columns_and_rows() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let destination = directory.path().join("selected.sql");
|
||||
let mut request = request("sql", None, None);
|
||||
request.file_path = destination.to_string_lossy().into_owned();
|
||||
request.export_column_types = Some(vec![Some("int".into()), Some("text".into()), Some("bigint".into())]);
|
||||
request.selected_columns = Some(vec![crate::types::SqlExportColumnSelection {
|
||||
source_index: 2,
|
||||
name: "id".into(),
|
||||
name_occurrence: 1,
|
||||
}]);
|
||||
let mut writer = SqlInsertWriter::create(&request).unwrap();
|
||||
writer
|
||||
.set_columns(
|
||||
vec!["id".into(), "name".into(), "id".into()],
|
||||
&["int".into(), "text".into(), "bigint".into()],
|
||||
&[],
|
||||
&request,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(writer.column_types, vec![Some("bigint".into())]);
|
||||
writer.write_row(vec![json!(1), json!("Ada"), json!(20)], None).unwrap();
|
||||
writer.write_row(vec![json!(2), json!("Lin"), Value::Null], None).unwrap();
|
||||
writer.finish().unwrap();
|
||||
let output = std::fs::read_to_string(destination).unwrap();
|
||||
assert!(output.contains("(\"id\") VALUES\n(20),\n(NULL);"), "{output}");
|
||||
assert!(!output.contains("name"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_insert_writer_aligns_request_types_and_identity_after_header_reorder() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let destination = directory.path().join("identity.sql");
|
||||
let mut request =
|
||||
sqlserver_identity_export_request(&destination, Some(vec![Some("identity(1,1)".into()), None]));
|
||||
request.selected_columns = Some(vec![crate::types::SqlExportColumnSelection {
|
||||
source_index: 0,
|
||||
name: "table_id".into(),
|
||||
name_occurrence: 0,
|
||||
}]);
|
||||
let mut writer = SqlInsertWriter::create(&request).unwrap();
|
||||
writer
|
||||
.set_columns(
|
||||
vec!["table_name".into(), "table_id".into()],
|
||||
&["nvarchar".into(), "int".into()],
|
||||
&[],
|
||||
&request,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(writer.column_types, vec![Some("int".into())]);
|
||||
assert_eq!(writer.column_extras, vec![Some("identity(1,1)".into())]);
|
||||
writer.write_row(vec![json!("Ada"), json!(23)], None).unwrap();
|
||||
writer.finish().unwrap();
|
||||
let output = std::fs::read_to_string(destination).unwrap();
|
||||
assert!(output.contains("SET IDENTITY_INSERT [dbo].[gen_table] ON;"));
|
||||
assert!(output.contains("([table_id]) VALUES (23);"));
|
||||
assert!(!output.contains("table_name"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_insert_writer_rejects_empty_selection_without_replacing_destination() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let destination = directory.path().join("preserved.sql");
|
||||
std::fs::write(&destination, "original").unwrap();
|
||||
let mut request = sqlserver_identity_export_request(&destination, None);
|
||||
request.selected_columns = Some(Vec::new());
|
||||
let mut writer = SqlInsertWriter::create(&request).unwrap();
|
||||
assert!(writer.set_columns(vec!["id".into()], &["int".into()], &[], &request).is_err());
|
||||
drop(writer);
|
||||
assert_eq!(std::fs::read_to_string(destination).unwrap(), "original");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_column_remap_keeps_duplicate_occurrences_across_pages() {
|
||||
let remap =
|
||||
export_column_remap(&["id".into(), "id".into(), "name".into()], &["id".into(), "name".into(), "id".into()]);
|
||||
assert_eq!(
|
||||
realign_page_cells(&remap, vec![json!(1), json!(2), json!("Ada")]),
|
||||
vec![json!(1), json!("Ada"), json!(2)]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_insert_writer_uses_one_batched_statement_by_default() {
|
||||
let output = rendered_sql_insert_output(SqlInsertMode::Batch);
|
||||
|
||||
@@ -22,7 +22,7 @@ use crate::data_grid_sql::extra_is_auto_generated;
|
||||
pub use crate::database_export::ExportStatus;
|
||||
use crate::database_export::{
|
||||
build_export_insert_statements_excluding_with_dialect, is_export_cancelled, is_internal_export_column,
|
||||
BuildExportInsertStatementsOptions, SqlInsertDialect, SqlInsertMode,
|
||||
BuildExportInsertStatementsOptions, SqlExportProjection, SqlInsertDialect, SqlInsertMode,
|
||||
};
|
||||
use crate::db::agent_driver::AgentTableReadStartParams;
|
||||
use crate::models::connection::DatabaseType;
|
||||
@@ -89,6 +89,8 @@ pub struct TableExportRequest {
|
||||
pub csv_quote_mode: CsvQuoteMode,
|
||||
#[serde(default)]
|
||||
pub columns: Option<Vec<String>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selected_columns: Option<Vec<crate::types::SqlExportColumnSelection>>,
|
||||
#[serde(default)]
|
||||
pub column_types: Option<Vec<Option<String>>>,
|
||||
/// Column EXTRA metadata for `columns`, supplied by the data grid. SQL
|
||||
@@ -1376,6 +1378,7 @@ async fn try_export_native_table_stream(
|
||||
}
|
||||
"sql" => {
|
||||
let mut file = create_table_export_sql_writer(request)?;
|
||||
let projection = SqlExportProjection::resolve(col_names, request.selected_columns.as_deref())?;
|
||||
let mut pending_rows: Vec<Vec<Value>> = Vec::new();
|
||||
let mut wrote_statements = false;
|
||||
let mut flush_pending =
|
||||
@@ -1384,7 +1387,7 @@ async fn try_export_native_table_stream(
|
||||
return Ok(());
|
||||
}
|
||||
let statements = build_export_insert_statements_excluding_with_dialect(
|
||||
BuildExportInsertStatementsOptions {
|
||||
projection.project_insert_options(BuildExportInsertStatementsOptions {
|
||||
database_type: Some(*db_type),
|
||||
identifier_quote: request.identifier_quote.clone(),
|
||||
schema: request.schema.clone(),
|
||||
@@ -1397,7 +1400,7 @@ async fn try_export_native_table_stream(
|
||||
spatial_values: Vec::new(),
|
||||
rows: std::mem::take(pending_rows),
|
||||
batch_size: Some(request.insert_mode.batch_size(SQL_INSERT_BATCH_SIZE)),
|
||||
},
|
||||
}),
|
||||
&sql_export_excluded_columns(request, primary_keys, col_names, column_extras),
|
||||
request.insert_dialect,
|
||||
)?;
|
||||
@@ -1606,10 +1609,18 @@ async fn export_table_data_core_inner(
|
||||
(col_names, column_types, column_extras, primary_keys)
|
||||
};
|
||||
|
||||
let sql_projection = if request.format.eq_ignore_ascii_case("sql") {
|
||||
Some(SqlExportProjection::resolve(&col_names, request.selected_columns.as_deref())?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if col_names.is_empty() {
|
||||
return Err("No columns found for table".to_string());
|
||||
}
|
||||
ensure_sql_insert_export_types_supported(&request.format, &column_types)?;
|
||||
if let Some(projection) = &sql_projection {
|
||||
ensure_sql_insert_export_types_supported(&request.format, &projection.project(&column_types))?;
|
||||
}
|
||||
let query_col_names = table_export_query_columns(request, &sql_context, &col_names)?;
|
||||
|
||||
// Use keyset pagination when all PKs are in the selected (filtered) columns.
|
||||
@@ -2190,6 +2201,7 @@ async fn export_table_data_core_inner(
|
||||
}
|
||||
"sql" => {
|
||||
let mut wrote_statements = false;
|
||||
let projection = sql_projection.as_ref().ok_or_else(|| "Missing SQL export projection".to_string())?;
|
||||
|
||||
loop {
|
||||
if is_export_cancelled(&request.export_id).await {
|
||||
@@ -2234,7 +2246,7 @@ async fn export_table_data_core_inner(
|
||||
}
|
||||
|
||||
let statements = build_export_insert_statements_excluding_with_dialect(
|
||||
BuildExportInsertStatementsOptions {
|
||||
projection.project_insert_options(BuildExportInsertStatementsOptions {
|
||||
database_type: Some(db_type),
|
||||
identifier_quote: request.identifier_quote.clone(),
|
||||
schema: request.schema.clone(),
|
||||
@@ -2247,7 +2259,7 @@ async fn export_table_data_core_inner(
|
||||
spatial_values: result.spatial_values.clone(),
|
||||
rows: result.rows.clone(),
|
||||
batch_size: Some(request.insert_mode.batch_size(SQL_INSERT_BATCH_SIZE)),
|
||||
},
|
||||
}),
|
||||
&sql_export_excluded_columns(request, &primary_keys, &col_names, &column_extras),
|
||||
request.insert_dialect,
|
||||
)?;
|
||||
@@ -2452,6 +2464,7 @@ mod tests {
|
||||
insert_mode: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
columns: Some(vec!["id".to_string(), "name".to_string()]),
|
||||
selected_columns: None,
|
||||
column_types: Some(vec![Some("INTEGER".to_string()), Some("VARCHAR".to_string())]),
|
||||
column_extras: None,
|
||||
primary_keys: Some(vec!["id".to_string()]),
|
||||
@@ -2620,6 +2633,7 @@ mod tests {
|
||||
csv_quote_mode: CsvQuoteMode::All,
|
||||
exclude_primary_keys: false,
|
||||
columns: None,
|
||||
selected_columns: None,
|
||||
column_types: None,
|
||||
column_extras: None,
|
||||
primary_keys: None,
|
||||
@@ -2787,6 +2801,7 @@ mod tests {
|
||||
insert_mode: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
columns: None,
|
||||
selected_columns: None,
|
||||
column_types: None,
|
||||
column_extras: None,
|
||||
primary_keys: None,
|
||||
@@ -2849,6 +2864,7 @@ mod tests {
|
||||
insert_mode: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
columns: None,
|
||||
selected_columns: None,
|
||||
column_types: None,
|
||||
column_extras: None,
|
||||
primary_keys: None,
|
||||
@@ -2889,6 +2905,7 @@ mod tests {
|
||||
insert_mode: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
columns: None,
|
||||
selected_columns: None,
|
||||
column_types: None,
|
||||
column_extras: None,
|
||||
primary_keys: None,
|
||||
@@ -2924,6 +2941,7 @@ mod tests {
|
||||
insert_mode: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
columns: None,
|
||||
selected_columns: None,
|
||||
column_types: None,
|
||||
column_extras: None,
|
||||
primary_keys: None,
|
||||
@@ -2967,6 +2985,7 @@ mod tests {
|
||||
insert_mode: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
columns: None,
|
||||
selected_columns: None,
|
||||
column_types: None,
|
||||
column_extras: None,
|
||||
primary_keys: None,
|
||||
@@ -3022,6 +3041,7 @@ mod tests {
|
||||
insert_mode: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
columns: None,
|
||||
selected_columns: None,
|
||||
column_types: None,
|
||||
column_extras: None,
|
||||
primary_keys: None,
|
||||
@@ -3071,6 +3091,7 @@ mod tests {
|
||||
insert_mode: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
columns: None,
|
||||
selected_columns: None,
|
||||
column_types: None,
|
||||
column_extras: None,
|
||||
primary_keys: None,
|
||||
@@ -3148,6 +3169,7 @@ mod tests {
|
||||
insert_mode: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
columns: None,
|
||||
selected_columns: None,
|
||||
column_types: None,
|
||||
column_extras: None,
|
||||
primary_keys: None,
|
||||
@@ -3207,6 +3229,7 @@ mod tests {
|
||||
insert_mode: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
columns: None,
|
||||
selected_columns: None,
|
||||
column_types: None,
|
||||
column_extras: None,
|
||||
primary_keys: None,
|
||||
@@ -3268,6 +3291,7 @@ mod tests {
|
||||
insert_mode: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
columns: None,
|
||||
selected_columns: None,
|
||||
column_types: None,
|
||||
column_extras: None,
|
||||
primary_keys: None,
|
||||
@@ -3441,7 +3465,7 @@ mod tests {
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("build table export plain SQL test runtime")
|
||||
.block_on(run_external_driver_sql_export_without_split_writes_plain_sql());
|
||||
.block_on(run_external_driver_sql_export_without_split_writes_plain_sql(false));
|
||||
})
|
||||
.expect("spawn table export plain SQL test thread");
|
||||
if let Err(panic) = handle.join() {
|
||||
@@ -3450,7 +3474,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
async fn run_external_driver_sql_export_without_split_writes_plain_sql() {
|
||||
async fn run_external_driver_sql_export_without_split_writes_plain_sql(select_name: bool) {
|
||||
let fixture = external_driver_export_fixture(
|
||||
r#" case "$line" in
|
||||
*'"method":"executeQueryPage"'*)
|
||||
@@ -3467,6 +3491,13 @@ mod tests {
|
||||
let mut request = fixture.request.clone();
|
||||
request.format = "sql".to_string();
|
||||
request.file_path = output.to_string_lossy().into_owned();
|
||||
if select_name {
|
||||
request.selected_columns = Some(vec![crate::types::SqlExportColumnSelection {
|
||||
source_index: 1,
|
||||
name: "name".into(),
|
||||
name_occurrence: 0,
|
||||
}]);
|
||||
}
|
||||
|
||||
let terminal = Arc::new(std::sync::Mutex::new(None::<ExportStatus>));
|
||||
let captured = terminal.clone();
|
||||
@@ -3481,6 +3512,10 @@ mod tests {
|
||||
assert_eq!(sql.matches("INSERT INTO").count(), 1, "both rows should be exported: {sql}");
|
||||
assert!(sql.contains("Ada"));
|
||||
assert!(sql.contains("Grace"));
|
||||
if select_name {
|
||||
assert!(sql.contains("(name) VALUES"), "{sql}");
|
||||
assert!(!sql.contains("(id,"), "{sql}");
|
||||
}
|
||||
assert!(sql.ends_with(";\n"));
|
||||
// Without `split_max_mb` the fallback arm must keep producing a plain
|
||||
// SQL file, not a zip archive.
|
||||
@@ -3504,7 +3539,7 @@ mod tests {
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("build table export split SQL test runtime")
|
||||
.block_on(run_external_driver_sql_export_honors_split_max_mb_into_zip_parts());
|
||||
.block_on(run_external_driver_sql_export_honors_split_max_mb_into_zip_parts(false));
|
||||
})
|
||||
.expect("spawn table export split SQL test thread");
|
||||
if let Err(panic) = handle.join() {
|
||||
@@ -3513,7 +3548,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
async fn run_external_driver_sql_export_honors_split_max_mb_into_zip_parts() {
|
||||
async fn run_external_driver_sql_export_honors_split_max_mb_into_zip_parts(select_name: bool) {
|
||||
// Two cursor pages: 200 wide rows (~1.2MB of INSERT statements) then a
|
||||
// short 100-row page. The first page alone must exceed the 1MB part
|
||||
// threshold, forcing the second page into a new zip part, and every
|
||||
@@ -3548,6 +3583,13 @@ esac"#,
|
||||
request.format = "sql".to_string();
|
||||
request.file_path = zip_path.to_string_lossy().into_owned();
|
||||
request.split_max_mb = Some(crate::export_split_zip::MIN_SPLIT_PART_MAX_MB);
|
||||
if select_name {
|
||||
request.selected_columns = Some(vec![crate::types::SqlExportColumnSelection {
|
||||
source_index: 1,
|
||||
name: "name".into(),
|
||||
name_occurrence: 0,
|
||||
}]);
|
||||
}
|
||||
|
||||
let terminal = Arc::new(std::sync::Mutex::new(None::<ExportStatus>));
|
||||
let captured = terminal.clone();
|
||||
@@ -3608,7 +3650,13 @@ esac"#,
|
||||
// All 300 rows must be present across the parts: every row value is
|
||||
// `(n, 'xxxx…')`, so one `, '` occurrence per row.
|
||||
let combined: String = sql_parts.iter().map(|(_, contents)| contents.as_str()).collect();
|
||||
assert_eq!(combined.matches(", '").count(), 300);
|
||||
if select_name {
|
||||
assert!(combined.contains("(name) VALUES"));
|
||||
assert!(!combined.contains("(id,"));
|
||||
assert_eq!(combined.matches("('x").count(), 300);
|
||||
} else {
|
||||
assert_eq!(combined.matches(", '").count(), 300);
|
||||
}
|
||||
|
||||
let manifest = manifest.expect("split export must contain a manifest");
|
||||
assert_eq!(manifest["totalParts"].as_u64().unwrap() as usize, sql_parts.len());
|
||||
@@ -3618,6 +3666,24 @@ esac"#,
|
||||
cleanup_external_driver_export_fixture(fixture);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn external_driver_sql_export_selected_columns_preserve_cursor_pages_and_zip_parts() {
|
||||
let handle = std::thread::Builder::new()
|
||||
.name("table-export-selected-columns".into())
|
||||
.stack_size(8 * 1024 * 1024)
|
||||
.spawn(|| {
|
||||
tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap().block_on(async {
|
||||
run_external_driver_sql_export_without_split_writes_plain_sql(true).await;
|
||||
run_external_driver_sql_export_honors_split_max_mb_into_zip_parts(true).await;
|
||||
});
|
||||
})
|
||||
.unwrap();
|
||||
if let Err(panic) = handle.join() {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn external_driver_table_export_does_not_repeat_legacy_one_shot_results() {
|
||||
|
||||
Reference in new issue
Block a user