fix(doris): preserve aggregate-state bytes in query results
This commit is contained in:
1 parent
4bcab01489
commit
c75ce783df
17 files changed
+1066
-42
No files matched your search
@@ -9,6 +9,7 @@ use crate::data_grid_sql::{
|
||||
is_grid_insert_omitted_column, is_non_identity_generated_column, supports_relational_copy_predicates,
|
||||
DataGridCopyInsertStatementOptions, DataGridCopyUpdateStatementOptions, DataGridTableMeta,
|
||||
};
|
||||
use dbx_types::types::is_opaque_aggregate_state_type;
|
||||
use serde_json::Value;
|
||||
use std::collections::{hash_map::Entry, HashMap, HashSet};
|
||||
use std::io::Write;
|
||||
@@ -61,6 +62,7 @@ pub(super) fn write_sql_inserts(
|
||||
output: &mut dyn Write,
|
||||
) -> Result<WriteMetadata, DataGridExtractError> {
|
||||
ensure_sql_builder_budget(context)?;
|
||||
reject_selected_opaque_aggregate_states(context)?;
|
||||
let data = sql_selected_data(context, false)?;
|
||||
let statement = build_data_grid_copy_insert_statement(DataGridCopyInsertStatementOptions {
|
||||
database_type: context.request.database_type,
|
||||
@@ -90,6 +92,7 @@ pub(super) fn write_sql_updates(
|
||||
output: &mut dyn Write,
|
||||
) -> Result<WriteMetadata, DataGridExtractError> {
|
||||
ensure_sql_builder_budget(context)?;
|
||||
reject_selected_opaque_aggregate_states(context)?;
|
||||
let table_meta = context.request.table_meta.as_ref().ok_or_else(|| {
|
||||
DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::MissingTableMetadata,
|
||||
@@ -133,6 +136,16 @@ pub(super) fn write_sql_updates(
|
||||
Ok(sql_metadata(data.omitted_columns))
|
||||
}
|
||||
|
||||
fn reject_selected_opaque_aggregate_states(context: &ExtractContext<'_>) -> Result<(), DataGridExtractError> {
|
||||
if context.selected_column_info.iter().flatten().any(|info| is_opaque_aggregate_state_type(&info.data_type)) {
|
||||
return Err(DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::NoWritableColumns,
|
||||
"Doris aggregate-state columns are opaque and cannot be copied as automatic SQL; use an explicit Doris state function instead.",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_sql_builder_budget(context: &ExtractContext<'_>) -> Result<(), DataGridExtractError> {
|
||||
let value_bytes = context
|
||||
.request
|
||||
|
||||
@@ -1363,6 +1363,39 @@ fn sql_updates_require_table_primary_key_and_writable_columns() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_copy_reports_selected_opaque_aggregate_states_before_generic_builder_errors() {
|
||||
let opaque_column = DataGridColumnInfo {
|
||||
name: "v2".to_string(),
|
||||
data_type: "agg_state<sum(int)>".to_string(),
|
||||
is_nullable: true,
|
||||
is_primary_key: false,
|
||||
column_default: None,
|
||||
extra: None,
|
||||
};
|
||||
for extractor in [DataGridExtractorId::SqlInserts, DataGridExtractorId::SqlUpdates] {
|
||||
let mut request = request(extractor);
|
||||
request.columns = vec![column("v2", 0)];
|
||||
request.selected_column_indexes = vec![0];
|
||||
request.rows = vec![vec![json!("0x00ff")]];
|
||||
request.table_meta = Some(DataGridTableMeta {
|
||||
catalog: None,
|
||||
database: None,
|
||||
schema: None,
|
||||
table_name: "states".to_string(),
|
||||
primary_keys: Vec::new(),
|
||||
columns: Some(vec![opaque_column.clone()]),
|
||||
});
|
||||
|
||||
let error = extract_data_grid_selection(request).expect_err("opaque SQL copy must fail explicitly");
|
||||
assert_eq!(error.code, DataGridExtractErrorCode::NoWritableColumns);
|
||||
assert_eq!(
|
||||
error.message,
|
||||
"Doris aggregate-state columns are opaque and cannot be copied as automatic SQL; use an explicit Doris state function instead."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_requests_that_exceed_the_column_budget() {
|
||||
let mut request = request(DataGridExtractorId::Csv);
|
||||
|
||||
@@ -34,6 +34,7 @@ use crate::sql_dialect::{
|
||||
uses_single_row_insert_statements, uses_synthetic_row_id, uses_xugu_row_id, TablePaginationStrategy,
|
||||
};
|
||||
use crate::value_literals::{format_ch_array_sql_literal, format_pg_array_sql_literal};
|
||||
use dbx_types::types::is_opaque_aggregate_state_type;
|
||||
|
||||
const DBX_ROWID_COLUMN: &str = "__DBX_ROWID";
|
||||
pub const DBX_NEO4J_ELEMENT_ID_COLUMN: &str = "__DBX_ELEMENT_ID";
|
||||
@@ -441,6 +442,14 @@ pub fn build_data_grid_copy_update_statements(options: DataGridCopyUpdateStateme
|
||||
let primary_key_info =
|
||||
primary_keys.iter().map(|primary_key| column_info_for(column_info, primary_key)).collect::<Vec<_>>();
|
||||
|
||||
if writable_indexes
|
||||
.iter()
|
||||
.any(|(_, _, info)| info.is_some_and(|info| is_opaque_aggregate_state_type(&info.data_type)))
|
||||
|| primary_key_info.iter().any(|info| info.is_some_and(|info| is_opaque_aggregate_state_type(&info.data_type)))
|
||||
{
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
if writable_indexes.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
@@ -541,6 +550,13 @@ pub fn build_data_grid_copy_insert_statement(options: DataGridCopyInsertStatemen
|
||||
.cloned()
|
||||
.collect();
|
||||
|
||||
if insert_columns
|
||||
.iter()
|
||||
.any(|(_, _, info)| info.as_ref().is_some_and(|info| is_opaque_aggregate_state_type(&info.data_type)))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
if insert_columns.is_empty() || options.rows.is_empty() {
|
||||
return None;
|
||||
}
|
||||
@@ -1199,6 +1215,9 @@ fn build_neo4j_data_grid_column_distinct_values_sql(options: &DataGridColumnDist
|
||||
}
|
||||
|
||||
fn validate_data_grid_save(options: &DataGridSaveStatementOptions) -> Option<String> {
|
||||
if let Some(error) = validate_opaque_aggregate_state_write(options) {
|
||||
return Some(error);
|
||||
}
|
||||
if let Some(error) = validate_salesforce_id_column(options) {
|
||||
return Some(error);
|
||||
}
|
||||
@@ -1281,6 +1300,45 @@ fn validate_data_grid_save(options: &DataGridSaveStatementOptions) -> Option<Str
|
||||
None
|
||||
}
|
||||
|
||||
fn validate_opaque_aggregate_state_write(options: &DataGridSaveStatementOptions) -> Option<String> {
|
||||
let save_columns = effective_columns(options);
|
||||
let column_info = options.table_meta.columns.as_deref().unwrap_or(&[]);
|
||||
let opaque_indexes = save_columns
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, column)| {
|
||||
let column = column.as_deref()?;
|
||||
column_info_for(column_info, column)
|
||||
.is_some_and(|info| is_opaque_aggregate_state_type(&info.data_type))
|
||||
.then_some(index)
|
||||
})
|
||||
.collect::<HashSet<_>>();
|
||||
if opaque_indexes.is_empty() {
|
||||
return None;
|
||||
}
|
||||
if options.dirty_rows.iter().any(|(_, changes)| changes.iter().any(|(index, _)| opaque_indexes.contains(index)))
|
||||
|| options
|
||||
.new_rows
|
||||
.iter()
|
||||
.any(|row| opaque_indexes.iter().any(|index| row.get(*index).is_some_and(|value| !value.is_null())))
|
||||
{
|
||||
return Some("Doris aggregate-state columns are opaque and cannot be written automatically; use an explicit Doris state function instead.".to_string());
|
||||
}
|
||||
if options.table_meta.primary_keys.is_empty()
|
||||
&& (!options.dirty_rows.is_empty() || !options.deleted_rows.is_empty())
|
||||
&& options.dirty_rows.iter().map(|(index, _)| *index).chain(options.deleted_rows.iter().copied()).any(
|
||||
|row_index| {
|
||||
options.rows.get(row_index).is_some_and(|row| {
|
||||
opaque_indexes.iter().any(|index| row.get(*index).is_some_and(|value| !value.is_null()))
|
||||
})
|
||||
},
|
||||
)
|
||||
{
|
||||
return Some("Cannot safely update or delete a keyless row whose predicate would contain an opaque Doris aggregate-state value.".to_string());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn validate_existing_row_primary_keys(options: &DataGridSaveStatementOptions) -> Option<String> {
|
||||
let primary_keys = &options.table_meta.primary_keys;
|
||||
if primary_keys.is_empty() || (options.dirty_rows.is_empty() && options.deleted_rows.is_empty()) {
|
||||
@@ -10155,4 +10213,61 @@ mod tests {
|
||||
assert_eq!(result.validation_error, Some(r#"Column "LogTime" does not allow NULL."#.to_string()));
|
||||
assert!(result.statements.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opaque_aggregate_state_blocks_keyless_predicates_and_copy_sql() {
|
||||
let table_meta = DataGridTableMeta {
|
||||
catalog: None,
|
||||
database: Some("analytics".to_string()),
|
||||
schema: None,
|
||||
table_name: "states".to_string(),
|
||||
primary_keys: vec![],
|
||||
columns: Some(vec![
|
||||
column("name", "varchar", false, None),
|
||||
column("v2", "agg_state<sum(int)>", false, None),
|
||||
]),
|
||||
};
|
||||
let save = prepare_data_grid_save(DataGridSaveStatementOptions {
|
||||
database_type: Some(DatabaseType::Doris),
|
||||
identifier_quote: None,
|
||||
table_meta: table_meta.clone(),
|
||||
columns: vec!["name".to_string(), "v2".to_string()],
|
||||
source_columns: None,
|
||||
rows: vec![vec![json!("before"), json!("0x00ff")]],
|
||||
dirty_rows: vec![(0, vec![(0, json!("after"))])],
|
||||
deleted_rows: vec![],
|
||||
new_rows: vec![],
|
||||
include_database_name: false,
|
||||
});
|
||||
assert!(save.validation_error.as_deref().is_some_and(|error| error.contains("keyless row")));
|
||||
assert!(save.statements.is_empty());
|
||||
|
||||
assert!(build_data_grid_copy_insert_statement(DataGridCopyInsertStatementOptions {
|
||||
database_type: Some(DatabaseType::Doris),
|
||||
identifier_quote: None,
|
||||
table_meta: Some(table_meta.clone()),
|
||||
columns: vec!["name".to_string(), "v2".to_string()],
|
||||
column_types: None,
|
||||
source_columns: None,
|
||||
rows: vec![vec![json!("before"), json!("0x00ff")]],
|
||||
exclude_primary_keys: false,
|
||||
include_computed_columns: false,
|
||||
include_database_name: true,
|
||||
insert_mode: DataGridCopyInsertMode::Merged,
|
||||
})
|
||||
.is_none());
|
||||
|
||||
let mut keyed = table_meta;
|
||||
keyed.primary_keys = vec!["name".to_string()];
|
||||
assert!(build_data_grid_copy_update_statements(DataGridCopyUpdateStatementOptions {
|
||||
database_type: Some(DatabaseType::Doris),
|
||||
identifier_quote: None,
|
||||
table_meta: keyed,
|
||||
columns: vec!["name".to_string(), "v2".to_string()],
|
||||
source_columns: None,
|
||||
rows: vec![vec![json!("before"), json!("0x00ff")]],
|
||||
include_database_name: true,
|
||||
})
|
||||
.is_empty());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user