fix(doris): preserve aggregate-state bytes in query results
This commit is contained in:
1 parent
4bcab01489
commit
c75ce783df
17 files changed
+1066
-42
No files matched your search
@@ -43,6 +43,58 @@ static OCEANBASE_MYSQL_TABLE_OPTION_RE: std::sync::LazyLock<Regex> = std::sync::
|
||||
static MYSQL_COLLATE_CLAUSE_RE: std::sync::LazyLock<Regex> = std::sync::LazyLock::new(|| {
|
||||
Regex::new(r"(?i)\bCOLLATE\s*=?\s*([A-Za-z0-9_]+)\b").expect("valid MySQL COLLATE clause regex")
|
||||
});
|
||||
|
||||
pub async fn ensure_transfer_source_types_supported(
|
||||
state: &AppState,
|
||||
request: &TransferRequest,
|
||||
source_pool_key: &str,
|
||||
) -> Result<(), String> {
|
||||
if matches!(request.content, TransferContent::StructureOnly) {
|
||||
return Ok(());
|
||||
}
|
||||
let is_doris = {
|
||||
let configs = state.configs.read().await;
|
||||
configs
|
||||
.get(&request.source_connection_id)
|
||||
.is_some_and(|config| db::doris::is_native_profile(&config.db_type, config.driver_profile.as_deref()))
|
||||
};
|
||||
if !is_doris {
|
||||
return Ok(());
|
||||
}
|
||||
for table in &request.tables {
|
||||
let columns = get_columns_for_transfer(
|
||||
state,
|
||||
source_pool_key,
|
||||
&request.source_connection_id,
|
||||
&request.source_database,
|
||||
&request.source_schema,
|
||||
table,
|
||||
request.source_catalog.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
ensure_transfer_columns_supported(request, true, table, &columns)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_transfer_columns_supported(
|
||||
request: &TransferRequest,
|
||||
is_doris_source: bool,
|
||||
table: &str,
|
||||
columns: &[db::ColumnInfo],
|
||||
) -> Result<(), String> {
|
||||
if matches!(request.content, TransferContent::StructureOnly) || !is_doris_source {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(column) = columns.iter().find(|column| crate::types::is_opaque_aggregate_state_type(&column.data_type))
|
||||
{
|
||||
return Err(format!(
|
||||
"Data transfer does not support Doris aggregate-state column `{}` in table `{table}`; export a representation format or use explicit Doris state functions instead",
|
||||
column.name
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
// An inline FK constraint *definition line*: an optional `CONSTRAINT <name>`
|
||||
// prefix followed by `FOREIGN KEY (`. Anchored to the line start so column
|
||||
// definitions whose COMMENT/DEFAULT strings mention "foreign key" never match.
|
||||
@@ -8840,6 +8892,14 @@ pub async fn rename_tables_to_backup<F>(
|
||||
where
|
||||
F: FnMut(TransferProgress),
|
||||
{
|
||||
let source_pool_key = ensure_transfer_pool(
|
||||
state,
|
||||
&request.source_connection_id,
|
||||
&request.source_database,
|
||||
request.source_catalog.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
ensure_transfer_source_types_supported(state, request, &source_pool_key).await?;
|
||||
let total_tables = tables.len();
|
||||
|
||||
// Resolve target names first so the fail-fast check below sees the names that will
|
||||
@@ -9419,6 +9479,14 @@ where
|
||||
return Err(format!("No columns found for table {table}"));
|
||||
}
|
||||
|
||||
let is_doris_source = {
|
||||
let configs = state.configs.read().await;
|
||||
configs
|
||||
.get(&request.source_connection_id)
|
||||
.is_some_and(|config| db::doris::is_native_profile(&config.db_type, config.driver_profile.as_deref()))
|
||||
};
|
||||
ensure_transfer_columns_supported(request, is_doris_source, table, &columns)?;
|
||||
|
||||
let writable_columns = writable_transfer_columns(&columns, source_db_type, target_db_type);
|
||||
let default_rows_only = mysql_generated_only_transfer(&columns, source_db_type, target_db_type);
|
||||
if writable_columns.is_empty() && !default_rows_only {
|
||||
@@ -12621,6 +12689,36 @@ CREATE TABLE "Other"."prefix""Source"."NAME" ("ID" INT);"#;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_column_validation_rejects_opaque_state_for_actual_doris_table() {
|
||||
let request = test_transfer_request(vec!["first_state", "actual_state"]);
|
||||
let columns = vec![test_column("id", "int"), test_column("v2", "agg_state<group_concat(text)>")];
|
||||
|
||||
let error = ensure_transfer_columns_supported(&request, true, "actual_state", &columns).unwrap_err();
|
||||
|
||||
assert!(error.contains("`v2`"), "{error}");
|
||||
assert!(error.contains("`actual_state`"), "{error}");
|
||||
assert!(!error.contains("first_state"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_column_validation_is_doris_data_only_and_structure_aware() {
|
||||
let columns = vec![test_column("v2", "AGG_STATE<SUM(INT)>")];
|
||||
let data_request = test_transfer_request(vec!["states"]);
|
||||
assert!(ensure_transfer_columns_supported(&data_request, false, "states", &columns).is_ok());
|
||||
|
||||
let structure_request = TransferRequest { content: TransferContent::StructureOnly, ..data_request };
|
||||
assert!(ensure_transfer_columns_supported(&structure_request, true, "states", &columns).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_column_validation_allows_regular_doris_columns() {
|
||||
let request = test_transfer_request(vec!["items"]);
|
||||
let columns = vec![test_column("id", "bigint"), test_column("name", "varchar(64)")];
|
||||
|
||||
assert!(ensure_transfer_columns_supported(&request, true, "items", &columns).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_request_defaults_preserve_table_name_case() {
|
||||
let request: TransferRequest = serde_json::from_value(json!({
|
||||
|
||||
Reference in new issue
Block a user