fix(doris): preserve aggregate-state bytes in query results

This commit is contained in:
Guoyu Su authored and GitHub committed 2026-09-28 01:01:09 +08:00
1 parent 4bcab01489
commit c75ce783df
17 files changed
+1066 -42

No files matched your search

@@ -9,6 +9,7 @@ use crate::data_grid_sql::{
is_grid_insert_omitted_column, is_non_identity_generated_column, supports_relational_copy_predicates,
DataGridCopyInsertStatementOptions, DataGridCopyUpdateStatementOptions, DataGridTableMeta,
};
use dbx_types::types::is_opaque_aggregate_state_type;
use serde_json::Value;
use std::collections::{hash_map::Entry, HashMap, HashSet};
use std::io::Write;
@@ -61,6 +62,7 @@ pub(super) fn write_sql_inserts(
output: &mut dyn Write,
) -> Result<WriteMetadata, DataGridExtractError> {
ensure_sql_builder_budget(context)?;
reject_selected_opaque_aggregate_states(context)?;
let data = sql_selected_data(context, false)?;
let statement = build_data_grid_copy_insert_statement(DataGridCopyInsertStatementOptions {
database_type: context.request.database_type,
@@ -90,6 +92,7 @@ pub(super) fn write_sql_updates(
output: &mut dyn Write,
) -> Result<WriteMetadata, DataGridExtractError> {
ensure_sql_builder_budget(context)?;
reject_selected_opaque_aggregate_states(context)?;
let table_meta = context.request.table_meta.as_ref().ok_or_else(|| {
DataGridExtractError::new(
DataGridExtractErrorCode::MissingTableMetadata,
@@ -133,6 +136,16 @@ pub(super) fn write_sql_updates(
Ok(sql_metadata(data.omitted_columns))
}
fn reject_selected_opaque_aggregate_states(context: &ExtractContext<'_>) -> Result<(), DataGridExtractError> {
if context.selected_column_info.iter().flatten().any(|info| is_opaque_aggregate_state_type(&info.data_type)) {
return Err(DataGridExtractError::new(
DataGridExtractErrorCode::NoWritableColumns,
"Doris aggregate-state columns are opaque and cannot be copied as automatic SQL; use an explicit Doris state function instead.",
));
}
Ok(())
}
fn ensure_sql_builder_budget(context: &ExtractContext<'_>) -> Result<(), DataGridExtractError> {
let value_bytes = context
.request
@@ -1363,6 +1363,39 @@ fn sql_updates_require_table_primary_key_and_writable_columns() {
);
}
#[test]
fn sql_copy_reports_selected_opaque_aggregate_states_before_generic_builder_errors() {
let opaque_column = DataGridColumnInfo {
name: "v2".to_string(),
data_type: "agg_state<sum(int)>".to_string(),
is_nullable: true,
is_primary_key: false,
column_default: None,
extra: None,
};
for extractor in [DataGridExtractorId::SqlInserts, DataGridExtractorId::SqlUpdates] {
let mut request = request(extractor);
request.columns = vec![column("v2", 0)];
request.selected_column_indexes = vec![0];
request.rows = vec![vec![json!("0x00ff")]];
request.table_meta = Some(DataGridTableMeta {
catalog: None,
database: None,
schema: None,
table_name: "states".to_string(),
primary_keys: Vec::new(),
columns: Some(vec![opaque_column.clone()]),
});
let error = extract_data_grid_selection(request).expect_err("opaque SQL copy must fail explicitly");
assert_eq!(error.code, DataGridExtractErrorCode::NoWritableColumns);
assert_eq!(
error.message,
"Doris aggregate-state columns are opaque and cannot be copied as automatic SQL; use an explicit Doris state function instead."
);
}
}
#[test]
fn rejects_requests_that_exceed_the_column_budget() {
let mut request = request(DataGridExtractorId::Csv);
+115
View File
@@ -34,6 +34,7 @@ use crate::sql_dialect::{
uses_single_row_insert_statements, uses_synthetic_row_id, uses_xugu_row_id, TablePaginationStrategy,
};
use crate::value_literals::{format_ch_array_sql_literal, format_pg_array_sql_literal};
use dbx_types::types::is_opaque_aggregate_state_type;
const DBX_ROWID_COLUMN: &str = "__DBX_ROWID";
pub const DBX_NEO4J_ELEMENT_ID_COLUMN: &str = "__DBX_ELEMENT_ID";
@@ -441,6 +442,14 @@ pub fn build_data_grid_copy_update_statements(options: DataGridCopyUpdateStateme
let primary_key_info =
primary_keys.iter().map(|primary_key| column_info_for(column_info, primary_key)).collect::<Vec<_>>();
if writable_indexes
.iter()
.any(|(_, _, info)| info.is_some_and(|info| is_opaque_aggregate_state_type(&info.data_type)))
|| primary_key_info.iter().any(|info| info.is_some_and(|info| is_opaque_aggregate_state_type(&info.data_type)))
{
return Vec::new();
}
if writable_indexes.is_empty() {
return Vec::new();
}
@@ -541,6 +550,13 @@ pub fn build_data_grid_copy_insert_statement(options: DataGridCopyInsertStatemen
.cloned()
.collect();
if insert_columns
.iter()
.any(|(_, _, info)| info.as_ref().is_some_and(|info| is_opaque_aggregate_state_type(&info.data_type)))
{
return None;
}
if insert_columns.is_empty() || options.rows.is_empty() {
return None;
}
@@ -1199,6 +1215,9 @@ fn build_neo4j_data_grid_column_distinct_values_sql(options: &DataGridColumnDist
}
fn validate_data_grid_save(options: &DataGridSaveStatementOptions) -> Option<String> {
if let Some(error) = validate_opaque_aggregate_state_write(options) {
return Some(error);
}
if let Some(error) = validate_salesforce_id_column(options) {
return Some(error);
}
@@ -1281,6 +1300,45 @@ fn validate_data_grid_save(options: &DataGridSaveStatementOptions) -> Option<Str
None
}
fn validate_opaque_aggregate_state_write(options: &DataGridSaveStatementOptions) -> Option<String> {
let save_columns = effective_columns(options);
let column_info = options.table_meta.columns.as_deref().unwrap_or(&[]);
let opaque_indexes = save_columns
.iter()
.enumerate()
.filter_map(|(index, column)| {
let column = column.as_deref()?;
column_info_for(column_info, column)
.is_some_and(|info| is_opaque_aggregate_state_type(&info.data_type))
.then_some(index)
})
.collect::<HashSet<_>>();
if opaque_indexes.is_empty() {
return None;
}
if options.dirty_rows.iter().any(|(_, changes)| changes.iter().any(|(index, _)| opaque_indexes.contains(index)))
|| options
.new_rows
.iter()
.any(|row| opaque_indexes.iter().any(|index| row.get(*index).is_some_and(|value| !value.is_null())))
{
return Some("Doris aggregate-state columns are opaque and cannot be written automatically; use an explicit Doris state function instead.".to_string());
}
if options.table_meta.primary_keys.is_empty()
&& (!options.dirty_rows.is_empty() || !options.deleted_rows.is_empty())
&& options.dirty_rows.iter().map(|(index, _)| *index).chain(options.deleted_rows.iter().copied()).any(
|row_index| {
options.rows.get(row_index).is_some_and(|row| {
opaque_indexes.iter().any(|index| row.get(*index).is_some_and(|value| !value.is_null()))
})
},
)
{
return Some("Cannot safely update or delete a keyless row whose predicate would contain an opaque Doris aggregate-state value.".to_string());
}
None
}
fn validate_existing_row_primary_keys(options: &DataGridSaveStatementOptions) -> Option<String> {
let primary_keys = &options.table_meta.primary_keys;
if primary_keys.is_empty() || (options.dirty_rows.is_empty() && options.deleted_rows.is_empty()) {
@@ -10155,4 +10213,61 @@ mod tests {
assert_eq!(result.validation_error, Some(r#"Column "LogTime" does not allow NULL."#.to_string()));
assert!(result.statements.is_empty());
}
#[test]
fn opaque_aggregate_state_blocks_keyless_predicates_and_copy_sql() {
let table_meta = DataGridTableMeta {
catalog: None,
database: Some("analytics".to_string()),
schema: None,
table_name: "states".to_string(),
primary_keys: vec![],
columns: Some(vec![
column("name", "varchar", false, None),
column("v2", "agg_state<sum(int)>", false, None),
]),
};
let save = prepare_data_grid_save(DataGridSaveStatementOptions {
database_type: Some(DatabaseType::Doris),
identifier_quote: None,
table_meta: table_meta.clone(),
columns: vec!["name".to_string(), "v2".to_string()],
source_columns: None,
rows: vec![vec![json!("before"), json!("0x00ff")]],
dirty_rows: vec![(0, vec![(0, json!("after"))])],
deleted_rows: vec![],
new_rows: vec![],
include_database_name: false,
});
assert!(save.validation_error.as_deref().is_some_and(|error| error.contains("keyless row")));
assert!(save.statements.is_empty());
assert!(build_data_grid_copy_insert_statement(DataGridCopyInsertStatementOptions {
database_type: Some(DatabaseType::Doris),
identifier_quote: None,
table_meta: Some(table_meta.clone()),
columns: vec!["name".to_string(), "v2".to_string()],
column_types: None,
source_columns: None,
rows: vec![vec![json!("before"), json!("0x00ff")]],
exclude_primary_keys: false,
include_computed_columns: false,
include_database_name: true,
insert_mode: DataGridCopyInsertMode::Merged,
})
.is_none());
let mut keyed = table_meta;
keyed.primary_keys = vec!["name".to_string()];
assert!(build_data_grid_copy_update_statements(DataGridCopyUpdateStatementOptions {
database_type: Some(DatabaseType::Doris),
identifier_quote: None,
table_meta: keyed,
columns: vec!["name".to_string(), "v2".to_string()],
source_columns: None,
rows: vec![vec![json!("before"), json!("0x00ff")]],
include_database_name: true,
})
.is_empty());
}
}