feat(export): support column selection in SQL exports

Closes #9475
This commit is contained in:
t8y2 committed 2026-09-27 22:37:50 +00:00
1 parent e9998e2806
commit 0a6ab8d360
28 files changed
+923 -85

No files matched your search

+159 -1
View File
@@ -22,7 +22,7 @@ use crate::transfer::{
keyset_pagination_sql_with_identifier_quote, quote_identifier, quote_postgres_string_literal,
wrap_dameng_identity_insert_sql_for_table,
};
use crate::types::{ObjectSourceKind, SpatialColumn};
use crate::types::{ObjectSourceKind, SpatialColumn, SqlExportColumnSelection};
static EXPORT_CANCELLED: std::sync::LazyLock<RwLock<HashSet<String>>> =
std::sync::LazyLock::new(|| RwLock::new(HashSet::new()));
@@ -1444,6 +1444,164 @@ pub(crate) fn is_internal_export_column(database_type: Option<DatabaseType>, col
&& column.eq_ignore_ascii_case(crate::sql_dialect::DBX_ROWID_COLUMN)
}
#[derive(Debug, Clone)]
pub(crate) struct SqlExportProjection {
entries: Vec<(usize, usize)>,
}
impl SqlExportProjection {
pub(crate) fn resolve(
columns: &[String],
selected_columns: Option<&[SqlExportColumnSelection]>,
) -> Result<Self, String> {
let Some(selected_columns) = selected_columns else {
return Ok(Self { entries: (0..columns.len()).map(|index| (index, index)).collect() });
};
if selected_columns.is_empty() {
return Err("Select at least one column for SQL export.".to_string());
}
let mut entries = Vec::with_capacity(selected_columns.len());
let mut used = HashSet::with_capacity(selected_columns.len());
let mut indexes_by_name = HashMap::<&str, Vec<usize>>::new();
for (index, name) in columns.iter().enumerate() {
indexes_by_name.entry(name.as_str()).or_default().push(index);
}
for selected in selected_columns {
let current_index = indexes_by_name
.get(selected.name.as_str())
.and_then(|indexes| indexes.get(selected.name_occurrence))
.copied()
.ok_or_else(|| format!("Selected SQL export column was not found: {}", selected.name))?;
if !used.insert(current_index) {
return Err(format!("SQL export column was selected more than once: {}", selected.name));
}
entries.push((current_index, selected.source_index));
}
Ok(Self { entries })
}
pub(crate) fn project<T: Clone + Default>(&self, values: &[T]) -> Vec<T> {
self.entries.iter().map(|(current_index, _)| values.get(*current_index).cloned().unwrap_or_default()).collect()
}
pub(crate) fn project_request<T: Clone + Default>(&self, values: &[T]) -> Vec<T> {
self.entries.iter().map(|(_, request_index)| values.get(*request_index).cloned().unwrap_or_default()).collect()
}
pub(crate) fn project_owned<T: Default>(&self, mut values: Vec<T>) -> Vec<T> {
self.entries.iter().map(|(index, _)| values.get_mut(*index).map(std::mem::take).unwrap_or_default()).collect()
}
pub(crate) fn project_spatial_columns(&self, columns: &[SpatialColumn]) -> Vec<SpatialColumn> {
let spatial_by_index: HashMap<_, _> = columns.iter().map(|column| (column.column_index, column)).collect();
self.entries
.iter()
.enumerate()
.filter_map(|(projected_index, (current_index, _))| {
spatial_by_index
.get(current_index)
.map(|column| SpatialColumn { column_index: projected_index, srid: column.srid })
})
.collect()
}
pub(crate) fn project_insert_options(
&self,
mut options: BuildExportInsertStatementsOptions,
) -> BuildExportInsertStatementsOptions {
options.columns = self.project_owned(options.columns);
options.column_types = self.project_owned(options.column_types);
options.column_extras = self.project_owned(options.column_extras);
options.spatial_columns = self.project_spatial_columns(&options.spatial_columns);
options.spatial_values = options.spatial_values.into_iter().map(|row| self.project_owned(row)).collect();
options.rows = options.rows.into_iter().map(|row| self.project_owned(row)).collect();
options
}
}
#[cfg(test)]
mod sql_export_projection_tests {
use super::*;
use serde_json::json;
fn selection(index: usize, name: &str, occurrence: usize) -> SqlExportColumnSelection {
SqlExportColumnSelection { source_index: index, name: name.into(), name_occurrence: occurrence }
}
#[test]
fn omitted_selection_keeps_every_column_but_empty_selection_is_rejected() {
let columns = vec!["id".into(), "name".into()];
let projection = SqlExportProjection::resolve(&columns, None).unwrap();
assert_eq!(projection.project(&columns), columns);
assert!(SqlExportProjection::resolve(&columns, Some(&[])).unwrap_err().contains("at least one"));
}
#[test]
fn duplicate_names_follow_occurrence_after_reordering() {
let columns = vec!["id".into(), "id".into(), "name".into()];
let projection =
SqlExportProjection::resolve(&columns, Some(&[selection(3, "id", 1), selection(2, "name", 0)])).unwrap();
assert_eq!(projection.project(&[json!(1), json!(2), json!("Ada")]), vec![json!(2), json!("Ada")]);
assert_eq!(projection.project_request(&[0, 1, 2, 3]), vec![3, 2]);
}
#[test]
fn missing_and_repeated_column_identities_are_rejected() {
let columns = vec!["id".into()];
assert!(SqlExportProjection::resolve(&columns, Some(&[selection(0, "id", 1)]))
.unwrap_err()
.contains("not found"));
assert!(SqlExportProjection::resolve(&columns, Some(&[selection(0, "name", 0)]))
.unwrap_err()
.contains("not found"));
assert!(SqlExportProjection::resolve(&columns, Some(&[selection(0, "id", 0), selection(1, "id", 0)]))
.unwrap_err()
.contains("more than once"));
}
#[test]
fn insert_projection_aligns_types_extras_spatial_data_and_values() {
let columns = vec!["id".into(), "shape".into(), "blob".into(), "optional".into()];
let projection = SqlExportProjection::resolve(
&columns,
Some(&[selection(2, "blob", 0), selection(1, "shape", 0), selection(3, "optional", 0)]),
)
.unwrap();
let options = BuildExportInsertStatementsOptions {
database_type: Some(DatabaseType::Mysql),
identifier_quote: None,
schema: None,
table_name: Some("items".into()),
qualified_table_name: None,
columns: columns.clone(),
column_types: vec![Some("int".into()), Some("geometry".into()), Some("blob".into()), Some("text".into())],
column_extras: vec![Some("auto_increment".into()), None, None, None],
spatial_columns: vec![SpatialColumn { column_index: 1, srid: Some(4326) }],
spatial_values: vec![vec![None, Some(4326), None, None]],
rows: vec![vec![json!(1), json!("POINT(1 2)"), json!([0, 255]), Value::Null]],
batch_size: Some(100),
};
let projected = projection.project_insert_options(options);
assert_eq!(columns, vec!["id", "shape", "blob", "optional"]);
assert_eq!(projected.columns, vec!["blob", "shape", "optional"]);
assert_eq!(projected.column_types, vec![Some("blob".into()), Some("geometry".into()), Some("text".into())]);
assert_eq!(projected.column_extras, vec![None, None, None]);
assert_eq!(projected.spatial_columns[0].column_index, 1);
assert_eq!(projected.spatial_values, vec![vec![None, Some(4326), None]]);
assert_eq!(projected.rows, vec![vec![json!([0, 255]), json!("POINT(1 2)"), Value::Null]]);
}
#[test]
fn wide_duplicate_selection_preserves_every_position() {
let columns = vec!["value".to_string(); 10000];
let selected: Vec<_> = (0..columns.len()).map(|index| selection(index, "value", index)).collect();
let projection = SqlExportProjection::resolve(&columns, Some(&selected)).unwrap();
let values: Vec<_> = (0..columns.len()).collect();
assert_eq!(projection.project_owned(values.clone()), values);
}
}
fn is_postgres_tsvector_export_column(database_type: Option<DatabaseType>, column_type: Option<&str>) -> bool {
database_type == Some(DatabaseType::Postgres)
&& column_type
+205 -42
View File
@@ -16,7 +16,8 @@ use crate::csv_export::{
};
pub use crate::database_export::ExportStatus;
use crate::database_export::{
build_export_insert_statements_excluding, is_export_cancelled, BuildExportInsertStatementsOptions, SqlInsertMode,
build_export_insert_statements_excluding, is_export_cancelled, BuildExportInsertStatementsOptions,
SqlExportProjection, SqlInsertMode,
};
use crate::models::connection::DatabaseType;
use crate::query::{
@@ -112,6 +113,8 @@ pub struct QueryResultExportRequest {
/// Frontend sends these in original full-query column order.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub export_column_types: Option<Vec<Option<String>>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selected_columns: Option<Vec<crate::types::SqlExportColumnSelection>>,
/// Column EXTRA metadata (for example SQL Server/Dameng `identity`) for SQL
/// INSERT export, sent by the frontend from the result's table metadata.
/// Without it the exported INSERT writes identity values without
@@ -296,19 +299,29 @@ fn stream_export_was_cancelled(error: &str, token_cancelled: bool, export_cancel
/// Missing, `None`, or empty overrides infer only MySQL spatial types, which
/// preserves the historical literal formatting of other database types.
fn sql_insert_column_types(request: &QueryResultExportRequest, column_types: &[String]) -> Vec<Option<String>> {
resolve_sql_insert_column_types(
request.database_type,
column_types,
request.export_column_types.as_deref().unwrap_or_default(),
)
}
fn resolve_sql_insert_column_types(
database_type: DatabaseType,
column_types: &[String],
overrides: &[Option<String>],
) -> Vec<Option<String>> {
column_types
.iter()
.enumerate()
.map(|(index, inferred)| {
request
.export_column_types
.as_ref()
.and_then(|overrides| overrides.get(index))
overrides
.get(index)
.and_then(|override_type| override_type.as_deref())
.filter(|override_type| !override_type.is_empty())
.map(str::to_string)
.or_else(|| {
(request.database_type == DatabaseType::Mysql
(database_type == DatabaseType::Mysql
&& crate::database_export::is_mysql_spatial_export_type(inferred))
.then(|| inferred.clone())
})
@@ -337,6 +350,7 @@ struct SqlInsertWriter {
schema: Option<String>,
table_name: String,
identifier_quote: Option<String>,
projection: Option<SqlExportProjection>,
/// 生成 INSERT 时需要排除的列名,来自请求里的“不含主键”设置。
exclude_columns: Vec<String>,
}
@@ -426,6 +440,7 @@ impl SqlInsertWriter {
schema: request.schema.clone(),
table_name,
identifier_quote: request.identifier_quote.clone(),
projection: None,
exclude_columns: if request.exclude_primary_keys { request.primary_keys.clone() } else { Vec::new() },
})
}
@@ -439,18 +454,42 @@ impl SqlInsertWriter {
result_column_types: &[String],
spatial_columns: &[SpatialColumn],
request: &QueryResultExportRequest,
) {
self.column_types = sql_insert_column_types(request, result_column_types);
) -> Result<(), String> {
let projection = SqlExportProjection::resolve(&columns, request.selected_columns.as_deref())?;
self.column_types = if request.selected_columns.is_some() {
resolve_sql_insert_column_types(
request.database_type,
&projection.project(result_column_types),
&projection.project_request(request.export_column_types.as_deref().unwrap_or_default()),
)
} else {
sql_insert_column_types(request, result_column_types)
};
// Column extras come from the request and align with the result columns the
// same way `export_column_types` does; missing entries simply mean "unknown".
self.column_extras = request.export_column_extras.clone().unwrap_or_default();
self.spatial_columns = spatial_columns.to_vec();
self.columns = columns;
self.column_extras = projection.project_request(request.export_column_extras.as_deref().unwrap_or_default());
self.spatial_columns = projection.project_spatial_columns(spatial_columns);
self.columns = projection.project(&columns);
self.projection = request.selected_columns.as_ref().map(|_| projection);
Ok(())
}
fn write_row(&mut self, row: Vec<Value>, spatial_values: Option<Vec<Option<u32>>>) -> Result<(), String> {
let row = match &self.projection {
Some(projection) => projection.project_owned(row),
None => row,
};
let spatial_values = spatial_values.unwrap_or_default();
let spatial_values = if spatial_values.is_empty() {
spatial_values
} else {
match &self.projection {
Some(projection) => projection.project_owned(spatial_values),
None => spatial_values,
}
};
self.pending_rows.push(row);
self.pending_spatial_values.push(spatial_values.unwrap_or_default());
self.pending_spatial_values.push(spatial_values);
if self.insert_mode.flush_each_row() || self.pending_rows.len() >= SQL_INSERT_BATCH_SIZE {
self.flush_batch()?;
}
@@ -647,7 +686,20 @@ fn single_execution_row_bound(request: &QueryResultExportRequest) -> Option<usiz
/// page payload, so later pages may reorder, add, or drop columns; values must
/// follow their column name instead of their position in the page.
fn export_column_remap(page_columns: &[String], header_columns: &[String]) -> Vec<Option<usize>> {
header_columns.iter().map(|column| page_columns.iter().position(|page_column| page_column == column)).collect()
let mut occurrences = std::collections::HashMap::<&str, usize>::new();
let mut indexes_by_name = std::collections::HashMap::<&str, Vec<usize>>::new();
for (index, name) in page_columns.iter().enumerate() {
indexes_by_name.entry(name.as_str()).or_default().push(index);
}
header_columns
.iter()
.map(|column| {
let occurrence = occurrences.entry(column.as_str()).or_default();
let index = indexes_by_name.get(column.as_str()).and_then(|indexes| indexes.get(*occurrence)).copied();
*occurrence += 1;
index
})
.collect()
}
/// Reorders one page row (or a page-level cell list such as column types) into
@@ -1010,7 +1062,7 @@ async fn export_query_result_core_inner(
columns = result.columns.clone();
column_types = result.column_types.clone();
if let Some(writer) = sql_writer.as_mut() {
writer.set_columns(columns.clone(), &column_types, &result.spatial_columns, request);
writer.set_columns(columns.clone(), &column_types, &result.spatial_columns, request)?;
}
column_types.clone()
} else if result.columns == columns {
@@ -1247,7 +1299,7 @@ async fn try_export_postgres_query_result_stream(
columns = stream_columns;
temporal_column_types = column_types.clone();
if let Some(writer) = sql_writer.as_mut() {
writer.set_columns(columns.clone(), &column_types, &[], request);
writer.set_columns(columns.clone(), &column_types, &[], request)?;
} else if let Some(file) = text_file.as_mut() {
let header = format_text_export_header(format, &columns, request.csv_quote_mode);
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
@@ -1493,7 +1545,7 @@ async fn try_export_mysql_query_result_stream(
columns = stream_columns;
temporal_column_types = column_types.clone();
if let Some(writer) = sql_writer.as_mut() {
writer.set_columns(columns.clone(), &column_types, &[], request);
writer.set_columns(columns.clone(), &column_types, &[], request)?;
} else if let Some(file) = text_file.as_mut() {
let header = format_text_export_header(format, &columns, request.csv_quote_mode);
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
@@ -1718,7 +1770,7 @@ async fn try_export_clickhouse_query_result_stream(
columns = stream_columns;
temporal_column_types = column_types.clone();
if let Some(writer) = sql_writer.as_mut() {
writer.set_columns(columns.clone(), &column_types, &[], request);
writer.set_columns(columns.clone(), &column_types, &[], request)?;
} else if let Some(file) = text_file.as_mut() {
let header = format_text_export_header(format, &columns, request.csv_quote_mode);
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
@@ -1912,7 +1964,7 @@ async fn try_export_sqlserver_query_result_stream(
columns = stream_columns.to_vec();
temporal_column_types = column_types.to_vec();
if let Some(writer) = sql_writer.as_mut() {
writer.set_columns(columns.clone(), &temporal_column_types, &[], request);
writer.set_columns(columns.clone(), &temporal_column_types, &[], request)?;
} else if let Some(file) = text_file.as_mut() {
let header = format_text_export_header(format, &columns, request.csv_quote_mode);
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
@@ -2131,6 +2183,7 @@ mod tests {
date_time_format: None,
export_table_name: Some("gen_table".to_string()),
export_column_types: Some(vec![Some("int".to_string()), Some("nvarchar(200)".to_string())]),
selected_columns: None,
export_column_extras,
numeric_column_right_align: false,
column_comments: None,
@@ -2151,12 +2204,14 @@ mod tests {
sqlserver_identity_export_request(&file_path, Some(vec![Some("identity(1,1)".to_string()), None]));
let mut writer = SqlInsertWriter::create(&request).expect("create sql insert writer");
writer.set_columns(
vec!["table_id".to_string(), "table_name".to_string()],
&["int".to_string(), "nvarchar(200)".to_string()],
&[],
&request,
);
writer
.set_columns(
vec!["table_id".to_string(), "table_name".to_string()],
&["int".to_string(), "nvarchar(200)".to_string()],
&[],
&request,
)
.expect("set SQL export columns");
writer.write_row(vec![json!(23), json!("t_destype")], None).expect("write export row");
writer.finish().expect("finish export");
@@ -2173,12 +2228,14 @@ mod tests {
let request = sqlserver_identity_export_request(&file_path, None);
let mut writer = SqlInsertWriter::create(&request).expect("create sql insert writer");
writer.set_columns(
vec!["table_id".to_string(), "table_name".to_string()],
&["int".to_string(), "nvarchar(200)".to_string()],
&[],
&request,
);
writer
.set_columns(
vec!["table_id".to_string(), "table_name".to_string()],
&["int".to_string(), "nvarchar(200)".to_string()],
&[],
&request,
)
.expect("set SQL export columns");
writer.write_row(vec![json!(23), json!("t_destype")], None).expect("write export row");
writer.finish().expect("finish export");
@@ -2216,6 +2273,7 @@ mod tests {
date_time_format: None,
export_table_name: Some("users".to_string()),
export_column_types: None,
selected_columns: None,
export_column_extras: None,
numeric_column_right_align: false,
column_comments: None,
@@ -2228,12 +2286,14 @@ mod tests {
};
let mut writer = SqlInsertWriter::create(&request).expect("create sql insert writer");
writer.set_columns(
vec!["id".to_string(), "name".to_string()],
&["integer".to_string(), "text".to_string()],
&[],
&request,
);
writer
.set_columns(
vec!["id".to_string(), "name".to_string()],
&["integer".to_string(), "text".to_string()],
&[],
&request,
)
.expect("set SQL export columns");
writer.write_row(vec![json!(1), json!("Ada")], None).expect("write export row");
writer.finish().expect("finish export");
@@ -2299,6 +2359,7 @@ mod tests {
csv_quote_mode: CsvQuoteMode::All,
export_table_name: None,
export_column_types: None,
selected_columns: None,
export_column_extras: None,
numeric_column_right_align: false,
column_comments: None,
@@ -2319,12 +2380,14 @@ mod tests {
req.insert_mode = mode;
let mut writer = SqlInsertWriter::create(&req).expect("create SQL writer");
writer.set_columns(
vec!["id".to_string(), "name".to_string()],
&["int".to_string(), "text".to_string()],
&[],
&req,
);
writer
.set_columns(
vec!["id".to_string(), "name".to_string()],
&["int".to_string(), "text".to_string()],
&[],
&req,
)
.expect("set SQL export columns");
writer.write_row(vec![serde_json::json!(1), serde_json::json!("Ada")], None).expect("write first row");
writer.write_row(vec![serde_json::json!(2), serde_json::json!("Lin")], None).expect("write second row");
writer.finish().expect("finish SQL writer");
@@ -2341,6 +2404,106 @@ mod tests {
assert_eq!(decoded.insert_mode, SqlInsertMode::Batch);
}
#[test]
fn sql_export_column_selection_is_optional_and_round_trips() {
let mut serialized = serde_json::to_value(request("sql", None, None)).unwrap();
serialized.as_object_mut().unwrap().remove("selectedColumns");
let mut decoded: QueryResultExportRequest = serde_json::from_value(serialized).unwrap();
assert!(decoded.selected_columns.is_none());
decoded.selected_columns = Some(vec![crate::types::SqlExportColumnSelection {
source_index: 2,
name: "id".into(),
name_occurrence: 1,
}]);
let encoded = serde_json::to_value(&decoded).unwrap();
assert_eq!(encoded["selectedColumns"], json!([{ "sourceIndex": 2, "name": "id", "nameOccurrence": 1 }]));
let round_trip: QueryResultExportRequest = serde_json::from_value(encoded).unwrap();
assert_eq!(round_trip.selected_columns.unwrap()[0].name_occurrence, 1);
}
#[test]
fn sql_insert_writer_projects_duplicate_columns_and_rows() {
let directory = tempfile::tempdir().unwrap();
let destination = directory.path().join("selected.sql");
let mut request = request("sql", None, None);
request.file_path = destination.to_string_lossy().into_owned();
request.export_column_types = Some(vec![Some("int".into()), Some("text".into()), Some("bigint".into())]);
request.selected_columns = Some(vec![crate::types::SqlExportColumnSelection {
source_index: 2,
name: "id".into(),
name_occurrence: 1,
}]);
let mut writer = SqlInsertWriter::create(&request).unwrap();
writer
.set_columns(
vec!["id".into(), "name".into(), "id".into()],
&["int".into(), "text".into(), "bigint".into()],
&[],
&request,
)
.unwrap();
assert_eq!(writer.column_types, vec![Some("bigint".into())]);
writer.write_row(vec![json!(1), json!("Ada"), json!(20)], None).unwrap();
writer.write_row(vec![json!(2), json!("Lin"), Value::Null], None).unwrap();
writer.finish().unwrap();
let output = std::fs::read_to_string(destination).unwrap();
assert!(output.contains("(\"id\") VALUES\n(20),\n(NULL);"), "{output}");
assert!(!output.contains("name"));
}
#[test]
fn sql_insert_writer_aligns_request_types_and_identity_after_header_reorder() {
let directory = tempfile::tempdir().unwrap();
let destination = directory.path().join("identity.sql");
let mut request =
sqlserver_identity_export_request(&destination, Some(vec![Some("identity(1,1)".into()), None]));
request.selected_columns = Some(vec![crate::types::SqlExportColumnSelection {
source_index: 0,
name: "table_id".into(),
name_occurrence: 0,
}]);
let mut writer = SqlInsertWriter::create(&request).unwrap();
writer
.set_columns(
vec!["table_name".into(), "table_id".into()],
&["nvarchar".into(), "int".into()],
&[],
&request,
)
.unwrap();
assert_eq!(writer.column_types, vec![Some("int".into())]);
assert_eq!(writer.column_extras, vec![Some("identity(1,1)".into())]);
writer.write_row(vec![json!("Ada"), json!(23)], None).unwrap();
writer.finish().unwrap();
let output = std::fs::read_to_string(destination).unwrap();
assert!(output.contains("SET IDENTITY_INSERT [dbo].[gen_table] ON;"));
assert!(output.contains("([table_id]) VALUES (23);"));
assert!(!output.contains("table_name"));
}
#[test]
fn sql_insert_writer_rejects_empty_selection_without_replacing_destination() {
let directory = tempfile::tempdir().unwrap();
let destination = directory.path().join("preserved.sql");
std::fs::write(&destination, "original").unwrap();
let mut request = sqlserver_identity_export_request(&destination, None);
request.selected_columns = Some(Vec::new());
let mut writer = SqlInsertWriter::create(&request).unwrap();
assert!(writer.set_columns(vec!["id".into()], &["int".into()], &[], &request).is_err());
drop(writer);
assert_eq!(std::fs::read_to_string(destination).unwrap(), "original");
}
#[test]
fn export_column_remap_keeps_duplicate_occurrences_across_pages() {
let remap =
export_column_remap(&["id".into(), "id".into(), "name".into()], &["id".into(), "name".into(), "id".into()]);
assert_eq!(
realign_page_cells(&remap, vec![json!(1), json!(2), json!("Ada")]),
vec![json!(1), json!("Ada"), json!(2)]
);
}
#[test]
fn sql_insert_writer_uses_one_batched_statement_by_default() {
let output = rendered_sql_insert_output(SqlInsertMode::Batch);
+77 -11
View File
@@ -22,7 +22,7 @@ use crate::data_grid_sql::extra_is_auto_generated;
pub use crate::database_export::ExportStatus;
use crate::database_export::{
build_export_insert_statements_excluding_with_dialect, is_export_cancelled, is_internal_export_column,
BuildExportInsertStatementsOptions, SqlInsertDialect, SqlInsertMode,
BuildExportInsertStatementsOptions, SqlExportProjection, SqlInsertDialect, SqlInsertMode,
};
use crate::db::agent_driver::AgentTableReadStartParams;
use crate::models::connection::DatabaseType;
@@ -89,6 +89,8 @@ pub struct TableExportRequest {
pub csv_quote_mode: CsvQuoteMode,
#[serde(default)]
pub columns: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selected_columns: Option<Vec<crate::types::SqlExportColumnSelection>>,
#[serde(default)]
pub column_types: Option<Vec<Option<String>>>,
/// Column EXTRA metadata for `columns`, supplied by the data grid. SQL
@@ -1376,6 +1378,7 @@ async fn try_export_native_table_stream(
}
"sql" => {
let mut file = create_table_export_sql_writer(request)?;
let projection = SqlExportProjection::resolve(col_names, request.selected_columns.as_deref())?;
let mut pending_rows: Vec<Vec<Value>> = Vec::new();
let mut wrote_statements = false;
let mut flush_pending =
@@ -1384,7 +1387,7 @@ async fn try_export_native_table_stream(
return Ok(());
}
let statements = build_export_insert_statements_excluding_with_dialect(
BuildExportInsertStatementsOptions {
projection.project_insert_options(BuildExportInsertStatementsOptions {
database_type: Some(*db_type),
identifier_quote: request.identifier_quote.clone(),
schema: request.schema.clone(),
@@ -1397,7 +1400,7 @@ async fn try_export_native_table_stream(
spatial_values: Vec::new(),
rows: std::mem::take(pending_rows),
batch_size: Some(request.insert_mode.batch_size(SQL_INSERT_BATCH_SIZE)),
},
}),
&sql_export_excluded_columns(request, primary_keys, col_names, column_extras),
request.insert_dialect,
)?;
@@ -1606,10 +1609,18 @@ async fn export_table_data_core_inner(
(col_names, column_types, column_extras, primary_keys)
};
let sql_projection = if request.format.eq_ignore_ascii_case("sql") {
Some(SqlExportProjection::resolve(&col_names, request.selected_columns.as_deref())?)
} else {
None
};
if col_names.is_empty() {
return Err("No columns found for table".to_string());
}
ensure_sql_insert_export_types_supported(&request.format, &column_types)?;
if let Some(projection) = &sql_projection {
ensure_sql_insert_export_types_supported(&request.format, &projection.project(&column_types))?;
}
let query_col_names = table_export_query_columns(request, &sql_context, &col_names)?;
// Use keyset pagination when all PKs are in the selected (filtered) columns.
@@ -2190,6 +2201,7 @@ async fn export_table_data_core_inner(
}
"sql" => {
let mut wrote_statements = false;
let projection = sql_projection.as_ref().ok_or_else(|| "Missing SQL export projection".to_string())?;
loop {
if is_export_cancelled(&request.export_id).await {
@@ -2234,7 +2246,7 @@ async fn export_table_data_core_inner(
}
let statements = build_export_insert_statements_excluding_with_dialect(
BuildExportInsertStatementsOptions {
projection.project_insert_options(BuildExportInsertStatementsOptions {
database_type: Some(db_type),
identifier_quote: request.identifier_quote.clone(),
schema: request.schema.clone(),
@@ -2247,7 +2259,7 @@ async fn export_table_data_core_inner(
spatial_values: result.spatial_values.clone(),
rows: result.rows.clone(),
batch_size: Some(request.insert_mode.batch_size(SQL_INSERT_BATCH_SIZE)),
},
}),
&sql_export_excluded_columns(request, &primary_keys, &col_names, &column_extras),
request.insert_dialect,
)?;
@@ -2452,6 +2464,7 @@ mod tests {
insert_mode: Default::default(),
insert_dialect: Default::default(),
columns: Some(vec!["id".to_string(), "name".to_string()]),
selected_columns: None,
column_types: Some(vec![Some("INTEGER".to_string()), Some("VARCHAR".to_string())]),
column_extras: None,
primary_keys: Some(vec!["id".to_string()]),
@@ -2620,6 +2633,7 @@ mod tests {
csv_quote_mode: CsvQuoteMode::All,
exclude_primary_keys: false,
columns: None,
selected_columns: None,
column_types: None,
column_extras: None,
primary_keys: None,
@@ -2787,6 +2801,7 @@ mod tests {
insert_mode: Default::default(),
insert_dialect: Default::default(),
columns: None,
selected_columns: None,
column_types: None,
column_extras: None,
primary_keys: None,
@@ -2849,6 +2864,7 @@ mod tests {
insert_mode: Default::default(),
insert_dialect: Default::default(),
columns: None,
selected_columns: None,
column_types: None,
column_extras: None,
primary_keys: None,
@@ -2889,6 +2905,7 @@ mod tests {
insert_mode: Default::default(),
insert_dialect: Default::default(),
columns: None,
selected_columns: None,
column_types: None,
column_extras: None,
primary_keys: None,
@@ -2924,6 +2941,7 @@ mod tests {
insert_mode: Default::default(),
insert_dialect: Default::default(),
columns: None,
selected_columns: None,
column_types: None,
column_extras: None,
primary_keys: None,
@@ -2967,6 +2985,7 @@ mod tests {
insert_mode: Default::default(),
insert_dialect: Default::default(),
columns: None,
selected_columns: None,
column_types: None,
column_extras: None,
primary_keys: None,
@@ -3022,6 +3041,7 @@ mod tests {
insert_mode: Default::default(),
insert_dialect: Default::default(),
columns: None,
selected_columns: None,
column_types: None,
column_extras: None,
primary_keys: None,
@@ -3071,6 +3091,7 @@ mod tests {
insert_mode: Default::default(),
insert_dialect: Default::default(),
columns: None,
selected_columns: None,
column_types: None,
column_extras: None,
primary_keys: None,
@@ -3148,6 +3169,7 @@ mod tests {
insert_mode: Default::default(),
insert_dialect: Default::default(),
columns: None,
selected_columns: None,
column_types: None,
column_extras: None,
primary_keys: None,
@@ -3207,6 +3229,7 @@ mod tests {
insert_mode: Default::default(),
insert_dialect: Default::default(),
columns: None,
selected_columns: None,
column_types: None,
column_extras: None,
primary_keys: None,
@@ -3268,6 +3291,7 @@ mod tests {
insert_mode: Default::default(),
insert_dialect: Default::default(),
columns: None,
selected_columns: None,
column_types: None,
column_extras: None,
primary_keys: None,
@@ -3441,7 +3465,7 @@ mod tests {
.enable_all()
.build()
.expect("build table export plain SQL test runtime")
.block_on(run_external_driver_sql_export_without_split_writes_plain_sql());
.block_on(run_external_driver_sql_export_without_split_writes_plain_sql(false));
})
.expect("spawn table export plain SQL test thread");
if let Err(panic) = handle.join() {
@@ -3450,7 +3474,7 @@ mod tests {
}
#[cfg(unix)]
async fn run_external_driver_sql_export_without_split_writes_plain_sql() {
async fn run_external_driver_sql_export_without_split_writes_plain_sql(select_name: bool) {
let fixture = external_driver_export_fixture(
r#" case "$line" in
*'"method":"executeQueryPage"'*)
@@ -3467,6 +3491,13 @@ mod tests {
let mut request = fixture.request.clone();
request.format = "sql".to_string();
request.file_path = output.to_string_lossy().into_owned();
if select_name {
request.selected_columns = Some(vec![crate::types::SqlExportColumnSelection {
source_index: 1,
name: "name".into(),
name_occurrence: 0,
}]);
}
let terminal = Arc::new(std::sync::Mutex::new(None::<ExportStatus>));
let captured = terminal.clone();
@@ -3481,6 +3512,10 @@ mod tests {
assert_eq!(sql.matches("INSERT INTO").count(), 1, "both rows should be exported: {sql}");
assert!(sql.contains("Ada"));
assert!(sql.contains("Grace"));
if select_name {
assert!(sql.contains("(name) VALUES"), "{sql}");
assert!(!sql.contains("(id,"), "{sql}");
}
assert!(sql.ends_with(";\n"));
// Without `split_max_mb` the fallback arm must keep producing a plain
// SQL file, not a zip archive.
@@ -3504,7 +3539,7 @@ mod tests {
.enable_all()
.build()
.expect("build table export split SQL test runtime")
.block_on(run_external_driver_sql_export_honors_split_max_mb_into_zip_parts());
.block_on(run_external_driver_sql_export_honors_split_max_mb_into_zip_parts(false));
})
.expect("spawn table export split SQL test thread");
if let Err(panic) = handle.join() {
@@ -3513,7 +3548,7 @@ mod tests {
}
#[cfg(unix)]
async fn run_external_driver_sql_export_honors_split_max_mb_into_zip_parts() {
async fn run_external_driver_sql_export_honors_split_max_mb_into_zip_parts(select_name: bool) {
// Two cursor pages: 200 wide rows (~1.2MB of INSERT statements) then a
// short 100-row page. The first page alone must exceed the 1MB part
// threshold, forcing the second page into a new zip part, and every
@@ -3548,6 +3583,13 @@ esac"#,
request.format = "sql".to_string();
request.file_path = zip_path.to_string_lossy().into_owned();
request.split_max_mb = Some(crate::export_split_zip::MIN_SPLIT_PART_MAX_MB);
if select_name {
request.selected_columns = Some(vec![crate::types::SqlExportColumnSelection {
source_index: 1,
name: "name".into(),
name_occurrence: 0,
}]);
}
let terminal = Arc::new(std::sync::Mutex::new(None::<ExportStatus>));
let captured = terminal.clone();
@@ -3608,7 +3650,13 @@ esac"#,
// All 300 rows must be present across the parts: every row value is
// `(n, 'xxxx…')`, so one `, '` occurrence per row.
let combined: String = sql_parts.iter().map(|(_, contents)| contents.as_str()).collect();
assert_eq!(combined.matches(", '").count(), 300);
if select_name {
assert!(combined.contains("(name) VALUES"));
assert!(!combined.contains("(id,"));
assert_eq!(combined.matches("('x").count(), 300);
} else {
assert_eq!(combined.matches(", '").count(), 300);
}
let manifest = manifest.expect("split export must contain a manifest");
assert_eq!(manifest["totalParts"].as_u64().unwrap() as usize, sql_parts.len());
@@ -3618,6 +3666,24 @@ esac"#,
cleanup_external_driver_export_fixture(fixture);
}
#[cfg(unix)]
#[test]
fn external_driver_sql_export_selected_columns_preserve_cursor_pages_and_zip_parts() {
let handle = std::thread::Builder::new()
.name("table-export-selected-columns".into())
.stack_size(8 * 1024 * 1024)
.spawn(|| {
tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap().block_on(async {
run_external_driver_sql_export_without_split_writes_plain_sql(true).await;
run_external_driver_sql_export_honors_split_max_mb_into_zip_parts(true).await;
});
})
.unwrap();
if let Err(panic) = handle.join() {
std::panic::resume_unwind(panic);
}
}
#[cfg(unix)]
#[tokio::test]
async fn external_driver_table_export_does_not_repeat_legacy_one_shot_results() {