fix(doris): preserve aggregate-state bytes in query results

This commit is contained in:
Guoyu Su authored and GitHub committed 2026-09-28 01:01:09 +08:00
1 parent 4bcab01489
commit c75ce783df
17 files changed
+1066 -42

No files matched your search

+36 -1
View File
@@ -1241,7 +1241,6 @@ pub(crate) fn build_export_insert_statements_excluding_with_dialect(
if options.columns.is_empty() || options.rows.is_empty() {
return Ok(Vec::new());
}
let excluded_names: HashSet<String> =
exclude_columns.iter().map(|column| column.trim().to_ascii_uppercase()).collect();
let table = export_qualified_table_name(
@@ -1274,6 +1273,18 @@ pub(crate) fn build_export_insert_statements_excluding_with_dialect(
})
})
.collect::<Vec<_>>();
if insert_columns.iter().any(|(index, _, _)| {
options
.column_types
.get(*index)
.and_then(|column_type| column_type.as_deref())
.is_some_and(crate::types::is_opaque_aggregate_state_type)
}) {
return Err(
"SQL INSERT export does not support Doris aggregate-state columns; use a representation export for canonical hex bytes"
.to_string(),
);
}
if insert_columns.is_empty() {
// Only fail when the exclusion itself removed the last insertable column;
// emptiness caused by other omission rules (e.g. generated columns) keeps
@@ -6125,6 +6136,30 @@ mod tests {
);
}
#[test]
fn sql_insert_export_allows_explicitly_excluded_opaque_column() {
let statements = build_export_insert_statements_excluding(
BuildExportInsertStatementsOptions {
database_type: Some(DatabaseType::Doris),
identifier_quote: None,
schema: None,
table_name: Some("states".to_string()),
qualified_table_name: None,
columns: vec!["id".to_string(), "v2".to_string()],
column_types: vec![Some("integer".to_string()), Some("agg_state<sum(int)>".to_string())],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!(1), json!("0x00ff")]],
batch_size: Some(10),
},
&["v2".to_string()],
)
.unwrap();
assert_eq!(statements, vec!["INSERT INTO `states` (`id`) VALUES (1);"]);
}
#[test]
fn sql_insert_export_reports_error_when_excluding_leaves_no_columns() {
let result = build_export_insert_statements_excluding(
+24 -1
View File
@@ -33,7 +33,7 @@ use crate::transfer::{
count_sql_with_where_and_identifier_quote, execute_read_on_pool, execute_read_on_pool_with_max_rows,
keyset_pagination_sql_with_identifier_quote, pagination_sql_with_filter_order_and_identifier_quote,
};
use crate::types::QueryResult;
use crate::types::{is_opaque_aggregate_state_type, QueryResult};
use crate::xlsx_export::{finish_streaming_xlsx_workbook, start_streaming_xlsx_workbook_with_options};
const DEFAULT_BATCH_SIZE: usize = 10_000;
@@ -237,6 +237,18 @@ fn requested_mysql_sql_export_needs_column_metadata(database_type: DatabaseType,
database_type == DatabaseType::Mysql && format.eq_ignore_ascii_case("sql")
}
fn ensure_sql_insert_export_types_supported(format: &str, column_types: &[Option<String>]) -> Result<(), String> {
if format.eq_ignore_ascii_case("sql")
&& column_types.iter().flatten().any(|column_type| is_opaque_aggregate_state_type(column_type))
{
return Err(
"SQL INSERT export does not support Doris aggregate-state columns; use CSV, JSON, TXT, or XLSX to export their canonical hex representation"
.to_string(),
);
}
Ok(())
}
/// Column EXTRA values supplied by the data grid, filtered and aligned exactly
/// like [`resolve_requested_export_columns`] keeps name/type metadata aligned.
fn resolve_requested_column_extras_by_position(
@@ -1597,6 +1609,7 @@ async fn export_table_data_core_inner(
if col_names.is_empty() {
return Err("No columns found for table".to_string());
}
ensure_sql_insert_export_types_supported(&request.format, &column_types)?;
let query_col_names = table_export_query_columns(request, &sql_context, &col_names)?;
// Use keyset pagination when all PKs are in the selected (filtered) columns.
@@ -2334,6 +2347,16 @@ mod tests {
assert_eq!(serde_json::to_value(request).unwrap()["insertDialect"], json!("standard"));
}
#[test]
fn sql_insert_export_rejects_opaque_aggregate_states_only() {
let types = vec![Some("int".to_string()), Some("agg_state<group_concat(text)>".to_string())];
assert!(ensure_sql_insert_export_types_supported("sql", &types)
.unwrap_err()
.contains("does not support Doris aggregate-state"));
assert!(ensure_sql_insert_export_types_supported("csv", &types).is_ok());
assert!(ensure_sql_insert_export_types_supported("sql", &[Some("varbinary(32)".to_string())]).is_ok());
}
#[cfg(unix)]
struct ExternalDriverExportFixture {
state: AppState,
+98
View File
@@ -43,6 +43,58 @@ static OCEANBASE_MYSQL_TABLE_OPTION_RE: std::sync::LazyLock<Regex> = std::sync::
static MYSQL_COLLATE_CLAUSE_RE: std::sync::LazyLock<Regex> = std::sync::LazyLock::new(|| {
Regex::new(r"(?i)\bCOLLATE\s*=?\s*([A-Za-z0-9_]+)\b").expect("valid MySQL COLLATE clause regex")
});
pub async fn ensure_transfer_source_types_supported(
state: &AppState,
request: &TransferRequest,
source_pool_key: &str,
) -> Result<(), String> {
if matches!(request.content, TransferContent::StructureOnly) {
return Ok(());
}
let is_doris = {
let configs = state.configs.read().await;
configs
.get(&request.source_connection_id)
.is_some_and(|config| db::doris::is_native_profile(&config.db_type, config.driver_profile.as_deref()))
};
if !is_doris {
return Ok(());
}
for table in &request.tables {
let columns = get_columns_for_transfer(
state,
source_pool_key,
&request.source_connection_id,
&request.source_database,
&request.source_schema,
table,
request.source_catalog.as_deref(),
)
.await?;
ensure_transfer_columns_supported(request, true, table, &columns)?;
}
Ok(())
}
fn ensure_transfer_columns_supported(
request: &TransferRequest,
is_doris_source: bool,
table: &str,
columns: &[db::ColumnInfo],
) -> Result<(), String> {
if matches!(request.content, TransferContent::StructureOnly) || !is_doris_source {
return Ok(());
}
if let Some(column) = columns.iter().find(|column| crate::types::is_opaque_aggregate_state_type(&column.data_type))
{
return Err(format!(
"Data transfer does not support Doris aggregate-state column `{}` in table `{table}`; export a representation format or use explicit Doris state functions instead",
column.name
));
}
Ok(())
}
// An inline FK constraint *definition line*: an optional `CONSTRAINT <name>`
// prefix followed by `FOREIGN KEY (`. Anchored to the line start so column
// definitions whose COMMENT/DEFAULT strings mention "foreign key" never match.
@@ -8840,6 +8892,14 @@ pub async fn rename_tables_to_backup<F>(
where
F: FnMut(TransferProgress),
{
let source_pool_key = ensure_transfer_pool(
state,
&request.source_connection_id,
&request.source_database,
request.source_catalog.as_deref(),
)
.await?;
ensure_transfer_source_types_supported(state, request, &source_pool_key).await?;
let total_tables = tables.len();
// Resolve target names first so the fail-fast check below sees the names that will
@@ -9419,6 +9479,14 @@ where
return Err(format!("No columns found for table {table}"));
}
let is_doris_source = {
let configs = state.configs.read().await;
configs
.get(&request.source_connection_id)
.is_some_and(|config| db::doris::is_native_profile(&config.db_type, config.driver_profile.as_deref()))
};
ensure_transfer_columns_supported(request, is_doris_source, table, &columns)?;
let writable_columns = writable_transfer_columns(&columns, source_db_type, target_db_type);
let default_rows_only = mysql_generated_only_transfer(&columns, source_db_type, target_db_type);
if writable_columns.is_empty() && !default_rows_only {
@@ -12621,6 +12689,36 @@ CREATE TABLE "Other"."prefix""Source"."NAME" ("ID" INT);"#;
}
}
#[test]
fn transfer_column_validation_rejects_opaque_state_for_actual_doris_table() {
let request = test_transfer_request(vec!["first_state", "actual_state"]);
let columns = vec![test_column("id", "int"), test_column("v2", "agg_state<group_concat(text)>")];
let error = ensure_transfer_columns_supported(&request, true, "actual_state", &columns).unwrap_err();
assert!(error.contains("`v2`"), "{error}");
assert!(error.contains("`actual_state`"), "{error}");
assert!(!error.contains("first_state"), "{error}");
}
#[test]
fn transfer_column_validation_is_doris_data_only_and_structure_aware() {
let columns = vec![test_column("v2", "AGG_STATE<SUM(INT)>")];
let data_request = test_transfer_request(vec!["states"]);
assert!(ensure_transfer_columns_supported(&data_request, false, "states", &columns).is_ok());
let structure_request = TransferRequest { content: TransferContent::StructureOnly, ..data_request };
assert!(ensure_transfer_columns_supported(&structure_request, true, "states", &columns).is_ok());
}
#[test]
fn transfer_column_validation_allows_regular_doris_columns() {
let request = test_transfer_request(vec!["items"]);
let columns = vec![test_column("id", "bigint"), test_column("name", "varchar(64)")];
assert!(ensure_transfer_columns_supported(&request, true, "items", &columns).is_ok());
}
#[test]
fn transfer_request_defaults_preserve_table_name_case() {
let request: TransferRequest = serde_json::from_value(json!({