fix(export): 分页导出按列名对齐每一页,修复字段错位 (#9583)
Schemaless 引擎(Elasticsearch/Easysearch)的列集合是按每个响应里的文档现算的, 因此 select * 的不同分页可能给出不同的列顺序或列集合。查询结果导出把表头冻结在 第一页,却按位置写入后续页的行,导致第二页起的值落到错误的列里(#9522 反馈的 "导出字段错位",例如 beta 的值被写进 alpha 列)。 - 新增 export_column_remap/realign_page_cells/realign_page_rows:后续页按列名映射到 冻结表头,列名缺失时填 NULL,列顺序不同也能正确还原; - 列类型同样按列名对齐,保证按页的日期/时间格式化使用正确的类型; - spatial_values 随行一起对齐,保持与单元格的 SRID 对应关系; - 第一页仍是表头来源,新增列(表头里没有的列)与被删除的列都会被忽略,行为与写出 表头保持一致。 Co-authored-by: zipg <4047349+zipg@users.noreply.github.com>
This commit is contained in:
1 parent
687b988b7d
commit
0adef6e16a
1 file changed
+70
-3
@@ -630,6 +630,27 @@ fn single_execution_row_bound(request: &QueryResultExportRequest) -> Option<usiz
|
||||
}
|
||||
}
|
||||
|
||||
/// Maps the frozen export header onto the column list a page reported.
|
||||
/// Schemaless engines (Elasticsearch, Easysearch) derive the column set from the
|
||||
/// page payload, so later pages may reorder, add, or drop columns; values must
|
||||
/// follow their column name instead of their position in the page.
|
||||
fn export_column_remap(page_columns: &[String], header_columns: &[String]) -> Vec<Option<usize>> {
|
||||
header_columns.iter().map(|column| page_columns.iter().position(|page_column| page_column == column)).collect()
|
||||
}
|
||||
|
||||
/// Reorders one page row (or a page-level cell list such as column types) into
|
||||
/// the frozen header order (`remap` indexes the page), filling cells whose column
|
||||
/// the page did not report with the cell type's default (`NULL` for values, empty
|
||||
/// for column types).
|
||||
fn realign_page_cells<T: Clone + Default>(remap: &[Option<usize>], cells: Vec<T>) -> Vec<T> {
|
||||
remap.iter().map(|index| index.and_then(|index| cells.get(index)).cloned().unwrap_or_default()).collect()
|
||||
}
|
||||
|
||||
/// Applies [`realign_page_cells`] to every row of a page.
|
||||
fn realign_page_rows<T: Clone + Default>(remap: &[Option<usize>], rows: Vec<Vec<T>>) -> Vec<Vec<T>> {
|
||||
rows.into_iter().map(|row| realign_page_cells(remap, row)).collect()
|
||||
}
|
||||
|
||||
fn single_execution_page_limit(request: &QueryResultExportRequest, page_size: usize) -> Option<usize> {
|
||||
single_execution_row_bound(request).filter(|bound| *bound > 0 && *bound <= page_size.max(1))
|
||||
}
|
||||
@@ -973,13 +994,27 @@ async fn export_query_result_core_inner(
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if columns.is_empty() {
|
||||
let page_column_types = if columns.is_empty() {
|
||||
columns = result.columns.clone();
|
||||
column_types = result.column_types.clone();
|
||||
if let Some(writer) = sql_writer.as_mut() {
|
||||
writer.set_columns(columns.clone(), &column_types, &result.spatial_columns, request);
|
||||
}
|
||||
}
|
||||
column_types.clone()
|
||||
} else if result.columns == columns {
|
||||
column_types.clone()
|
||||
} else {
|
||||
// The export header is frozen by the first page, so every later page
|
||||
// must be aligned by column name; writing page rows positionally
|
||||
// dropped/duplicated values into the wrong columns.
|
||||
let remap = export_column_remap(&result.columns, &columns);
|
||||
let page_column_types = realign_page_cells(&remap, result.column_types.clone());
|
||||
result.rows = realign_page_rows(&remap, mem::take(&mut result.rows));
|
||||
if !result.spatial_values.is_empty() {
|
||||
result.spatial_values = realign_page_rows(&remap, mem::take(&mut result.spatial_values));
|
||||
}
|
||||
page_column_types
|
||||
};
|
||||
let fetched_row_count = result.rows.len();
|
||||
if result.rows.len() > this_page {
|
||||
result.rows.truncate(this_page);
|
||||
@@ -987,7 +1022,7 @@ async fn export_query_result_core_inner(
|
||||
let row_count = result.rows.len();
|
||||
let formatted_rows = crate::temporal_format::format_temporal_export_rows_with_string_types_cow(
|
||||
&result.rows,
|
||||
&column_types,
|
||||
&page_column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
);
|
||||
|
||||
@@ -1980,6 +2015,38 @@ mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn export_column_remap_follows_column_names_across_pages() {
|
||||
// Elasticsearch-style page drift: the second page reports `beta` where the
|
||||
// header has `alpha`, so the `alpha` cell must be filled with NULL instead
|
||||
// of the value that happens to sit at the same position.
|
||||
let header = vec!["id".to_string(), "alpha".to_string(), "_id".to_string()];
|
||||
let page = vec!["id".to_string(), "beta".to_string(), "_id".to_string()];
|
||||
let remap = export_column_remap(&page, &header);
|
||||
assert_eq!(remap, vec![Some(0), None, Some(2)]);
|
||||
let rows = vec![vec![json!(101), json!("B101"), json!("101")]];
|
||||
assert_eq!(realign_page_rows(&remap, rows), vec![vec![json!(101), Value::Null, json!("101")]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_column_remap_reorders_pages_that_report_columns_differently() {
|
||||
let header = vec!["id".to_string(), "alpha".to_string()];
|
||||
let page = vec!["alpha".to_string(), "id".to_string()];
|
||||
let remap = export_column_remap(&page, &header);
|
||||
assert_eq!(remap, vec![Some(1), Some(0)]);
|
||||
let rows = vec![vec![json!("A1"), json!(1)]];
|
||||
assert_eq!(realign_page_rows(&remap, rows), vec![vec![json!(1), json!("A1")]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_column_remap_keeps_identical_headers_untouched() {
|
||||
let header = vec!["id".to_string(), "alpha".to_string()];
|
||||
let remap = export_column_remap(&header, &header);
|
||||
assert_eq!(remap, vec![Some(0), Some(1)]);
|
||||
let rows = vec![vec![json!(1), json!("A1")]];
|
||||
assert_eq!(realign_page_rows(&remap, rows.clone()), rows);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn staged_export_target_preserves_existing_destination_on_discard_and_replace_failure() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
|
||||
Reference in new issue
Block a user