fix(doris): preserve aggregate-state bytes in query results
This commit is contained in:
1 parent
4bcab01489
commit
c75ce783df
17 files changed
+1066
-42
No files matched your search
@@ -1241,7 +1241,6 @@ pub(crate) fn build_export_insert_statements_excluding_with_dialect(
|
||||
if options.columns.is_empty() || options.rows.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let excluded_names: HashSet<String> =
|
||||
exclude_columns.iter().map(|column| column.trim().to_ascii_uppercase()).collect();
|
||||
let table = export_qualified_table_name(
|
||||
@@ -1274,6 +1273,18 @@ pub(crate) fn build_export_insert_statements_excluding_with_dialect(
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if insert_columns.iter().any(|(index, _, _)| {
|
||||
options
|
||||
.column_types
|
||||
.get(*index)
|
||||
.and_then(|column_type| column_type.as_deref())
|
||||
.is_some_and(crate::types::is_opaque_aggregate_state_type)
|
||||
}) {
|
||||
return Err(
|
||||
"SQL INSERT export does not support Doris aggregate-state columns; use a representation export for canonical hex bytes"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
if insert_columns.is_empty() {
|
||||
// Only fail when the exclusion itself removed the last insertable column;
|
||||
// emptiness caused by other omission rules (e.g. generated columns) keeps
|
||||
@@ -6125,6 +6136,30 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_insert_export_allows_explicitly_excluded_opaque_column() {
|
||||
let statements = build_export_insert_statements_excluding(
|
||||
BuildExportInsertStatementsOptions {
|
||||
database_type: Some(DatabaseType::Doris),
|
||||
identifier_quote: None,
|
||||
schema: None,
|
||||
table_name: Some("states".to_string()),
|
||||
qualified_table_name: None,
|
||||
columns: vec!["id".to_string(), "v2".to_string()],
|
||||
column_types: vec![Some("integer".to_string()), Some("agg_state<sum(int)>".to_string())],
|
||||
column_extras: Vec::new(),
|
||||
spatial_columns: Vec::new(),
|
||||
spatial_values: Vec::new(),
|
||||
rows: vec![vec![json!(1), json!("0x00ff")]],
|
||||
batch_size: Some(10),
|
||||
},
|
||||
&["v2".to_string()],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(statements, vec!["INSERT INTO `states` (`id`) VALUES (1);"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_insert_export_reports_error_when_excluding_leaves_no_columns() {
|
||||
let result = build_export_insert_statements_excluding(
|
||||
|
||||
@@ -33,7 +33,7 @@ use crate::transfer::{
|
||||
count_sql_with_where_and_identifier_quote, execute_read_on_pool, execute_read_on_pool_with_max_rows,
|
||||
keyset_pagination_sql_with_identifier_quote, pagination_sql_with_filter_order_and_identifier_quote,
|
||||
};
|
||||
use crate::types::QueryResult;
|
||||
use crate::types::{is_opaque_aggregate_state_type, QueryResult};
|
||||
use crate::xlsx_export::{finish_streaming_xlsx_workbook, start_streaming_xlsx_workbook_with_options};
|
||||
|
||||
const DEFAULT_BATCH_SIZE: usize = 10_000;
|
||||
@@ -237,6 +237,18 @@ fn requested_mysql_sql_export_needs_column_metadata(database_type: DatabaseType,
|
||||
database_type == DatabaseType::Mysql && format.eq_ignore_ascii_case("sql")
|
||||
}
|
||||
|
||||
fn ensure_sql_insert_export_types_supported(format: &str, column_types: &[Option<String>]) -> Result<(), String> {
|
||||
if format.eq_ignore_ascii_case("sql")
|
||||
&& column_types.iter().flatten().any(|column_type| is_opaque_aggregate_state_type(column_type))
|
||||
{
|
||||
return Err(
|
||||
"SQL INSERT export does not support Doris aggregate-state columns; use CSV, JSON, TXT, or XLSX to export their canonical hex representation"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Column EXTRA values supplied by the data grid, filtered and aligned exactly
|
||||
/// like [`resolve_requested_export_columns`] keeps name/type metadata aligned.
|
||||
fn resolve_requested_column_extras_by_position(
|
||||
@@ -1597,6 +1609,7 @@ async fn export_table_data_core_inner(
|
||||
if col_names.is_empty() {
|
||||
return Err("No columns found for table".to_string());
|
||||
}
|
||||
ensure_sql_insert_export_types_supported(&request.format, &column_types)?;
|
||||
let query_col_names = table_export_query_columns(request, &sql_context, &col_names)?;
|
||||
|
||||
// Use keyset pagination when all PKs are in the selected (filtered) columns.
|
||||
@@ -2334,6 +2347,16 @@ mod tests {
|
||||
assert_eq!(serde_json::to_value(request).unwrap()["insertDialect"], json!("standard"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_insert_export_rejects_opaque_aggregate_states_only() {
|
||||
let types = vec![Some("int".to_string()), Some("agg_state<group_concat(text)>".to_string())];
|
||||
assert!(ensure_sql_insert_export_types_supported("sql", &types)
|
||||
.unwrap_err()
|
||||
.contains("does not support Doris aggregate-state"));
|
||||
assert!(ensure_sql_insert_export_types_supported("csv", &types).is_ok());
|
||||
assert!(ensure_sql_insert_export_types_supported("sql", &[Some("varbinary(32)".to_string())]).is_ok());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
struct ExternalDriverExportFixture {
|
||||
state: AppState,
|
||||
|
||||
@@ -43,6 +43,58 @@ static OCEANBASE_MYSQL_TABLE_OPTION_RE: std::sync::LazyLock<Regex> = std::sync::
|
||||
static MYSQL_COLLATE_CLAUSE_RE: std::sync::LazyLock<Regex> = std::sync::LazyLock::new(|| {
|
||||
Regex::new(r"(?i)\bCOLLATE\s*=?\s*([A-Za-z0-9_]+)\b").expect("valid MySQL COLLATE clause regex")
|
||||
});
|
||||
|
||||
pub async fn ensure_transfer_source_types_supported(
|
||||
state: &AppState,
|
||||
request: &TransferRequest,
|
||||
source_pool_key: &str,
|
||||
) -> Result<(), String> {
|
||||
if matches!(request.content, TransferContent::StructureOnly) {
|
||||
return Ok(());
|
||||
}
|
||||
let is_doris = {
|
||||
let configs = state.configs.read().await;
|
||||
configs
|
||||
.get(&request.source_connection_id)
|
||||
.is_some_and(|config| db::doris::is_native_profile(&config.db_type, config.driver_profile.as_deref()))
|
||||
};
|
||||
if !is_doris {
|
||||
return Ok(());
|
||||
}
|
||||
for table in &request.tables {
|
||||
let columns = get_columns_for_transfer(
|
||||
state,
|
||||
source_pool_key,
|
||||
&request.source_connection_id,
|
||||
&request.source_database,
|
||||
&request.source_schema,
|
||||
table,
|
||||
request.source_catalog.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
ensure_transfer_columns_supported(request, true, table, &columns)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_transfer_columns_supported(
|
||||
request: &TransferRequest,
|
||||
is_doris_source: bool,
|
||||
table: &str,
|
||||
columns: &[db::ColumnInfo],
|
||||
) -> Result<(), String> {
|
||||
if matches!(request.content, TransferContent::StructureOnly) || !is_doris_source {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(column) = columns.iter().find(|column| crate::types::is_opaque_aggregate_state_type(&column.data_type))
|
||||
{
|
||||
return Err(format!(
|
||||
"Data transfer does not support Doris aggregate-state column `{}` in table `{table}`; export a representation format or use explicit Doris state functions instead",
|
||||
column.name
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
// An inline FK constraint *definition line*: an optional `CONSTRAINT <name>`
|
||||
// prefix followed by `FOREIGN KEY (`. Anchored to the line start so column
|
||||
// definitions whose COMMENT/DEFAULT strings mention "foreign key" never match.
|
||||
@@ -8840,6 +8892,14 @@ pub async fn rename_tables_to_backup<F>(
|
||||
where
|
||||
F: FnMut(TransferProgress),
|
||||
{
|
||||
let source_pool_key = ensure_transfer_pool(
|
||||
state,
|
||||
&request.source_connection_id,
|
||||
&request.source_database,
|
||||
request.source_catalog.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
ensure_transfer_source_types_supported(state, request, &source_pool_key).await?;
|
||||
let total_tables = tables.len();
|
||||
|
||||
// Resolve target names first so the fail-fast check below sees the names that will
|
||||
@@ -9419,6 +9479,14 @@ where
|
||||
return Err(format!("No columns found for table {table}"));
|
||||
}
|
||||
|
||||
let is_doris_source = {
|
||||
let configs = state.configs.read().await;
|
||||
configs
|
||||
.get(&request.source_connection_id)
|
||||
.is_some_and(|config| db::doris::is_native_profile(&config.db_type, config.driver_profile.as_deref()))
|
||||
};
|
||||
ensure_transfer_columns_supported(request, is_doris_source, table, &columns)?;
|
||||
|
||||
let writable_columns = writable_transfer_columns(&columns, source_db_type, target_db_type);
|
||||
let default_rows_only = mysql_generated_only_transfer(&columns, source_db_type, target_db_type);
|
||||
if writable_columns.is_empty() && !default_rows_only {
|
||||
@@ -12621,6 +12689,36 @@ CREATE TABLE "Other"."prefix""Source"."NAME" ("ID" INT);"#;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_column_validation_rejects_opaque_state_for_actual_doris_table() {
|
||||
let request = test_transfer_request(vec!["first_state", "actual_state"]);
|
||||
let columns = vec![test_column("id", "int"), test_column("v2", "agg_state<group_concat(text)>")];
|
||||
|
||||
let error = ensure_transfer_columns_supported(&request, true, "actual_state", &columns).unwrap_err();
|
||||
|
||||
assert!(error.contains("`v2`"), "{error}");
|
||||
assert!(error.contains("`actual_state`"), "{error}");
|
||||
assert!(!error.contains("first_state"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_column_validation_is_doris_data_only_and_structure_aware() {
|
||||
let columns = vec![test_column("v2", "AGG_STATE<SUM(INT)>")];
|
||||
let data_request = test_transfer_request(vec!["states"]);
|
||||
assert!(ensure_transfer_columns_supported(&data_request, false, "states", &columns).is_ok());
|
||||
|
||||
let structure_request = TransferRequest { content: TransferContent::StructureOnly, ..data_request };
|
||||
assert!(ensure_transfer_columns_supported(&structure_request, true, "states", &columns).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_column_validation_allows_regular_doris_columns() {
|
||||
let request = test_transfer_request(vec!["items"]);
|
||||
let columns = vec![test_column("id", "bigint"), test_column("name", "varchar(64)")];
|
||||
|
||||
assert!(ensure_transfer_columns_supported(&request, true, "items", &columns).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_request_defaults_preserve_table_name_case() {
|
||||
let request: TransferRequest = serde_json::from_value(json!({
|
||||
|
||||
Reference in new issue
Block a user