From 4e0bec13f1b1e222a65018fec4c920f146068212 Mon Sep 17 00:00:00 2001 From: Freedom <48708512+xddcode@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:30:57 +0800 Subject: [PATCH] fix(mysql): preserve group concat metadata limit --- crates/dbx-core/src/db/mysql.rs | 125 +++++++++++++++-------- crates/dbx-core/src/models/connection.rs | 10 ++ 2 files changed, 92 insertions(+), 43 deletions(-) diff --git a/crates/dbx-core/src/db/mysql.rs b/crates/dbx-core/src/db/mysql.rs index 9f3fc1a313..7252319a65 100644 --- a/crates/dbx-core/src/db/mysql.rs +++ b/crates/dbx-core/src/db/mysql.rs @@ -598,8 +598,11 @@ enum MySqlSetupMode { } impl MySqlSetupMode { - fn set_group_concat_max_len(self) -> bool { - self == Self::Standard + fn group_concat_max_len_query(self) -> Option<&'static str> { + match self { + Self::Standard => Some("SET SESSION group_concat_max_len = 1048576"), + Self::Compatible => None, + } } } @@ -617,12 +620,10 @@ async fn verify_pool_connection_with_setup_fallback( match verify_pool_connection(&pool, timeout).await { Ok(()) => Ok(pool), Err(err) => { - let Some(fallback_mode) = mysql_setup_mode_retry_without_group_concat(setup_mode, &err) else { + let Some(fallback_mode) = mysql_group_concat_setup_fallback_mode(setup_mode, &err) else { return Err(err); }; - log::info!( - "MySQL server rejected optional group_concat_max_len setup; retrying without that session setting" - ); + log::info!("MySQL server rejected group_concat_max_len setup syntax; retrying with {fallback_mode:?} mode"); let fallback_pool = create_pool( url, ca_cert_path, @@ -637,11 +638,16 @@ async fn verify_pool_connection_with_setup_fallback( } } -fn mysql_setup_mode_retry_without_group_concat(setup_mode: MySqlSetupMode, error: &str) -> Option { - // group_concat_max_len improves real MySQL metadata reads, but some MySQL-compatible proxies reject the variable. - if setup_mode.set_group_concat_max_len() && mysql_error_should_retry_without_group_concat_max_len(error) { +fn mysql_group_concat_setup_fallback_mode(setup_mode: MySqlSetupMode, error: &str) -> Option { + if setup_mode != MySqlSetupMode::Standard { + return None; + } + + let lower = error.to_ascii_lowercase(); + if lower.contains("group_concat_max_len") && (lower.contains("1193") || lower.contains("unknown system variable")) { return Some(MySqlSetupMode::Compatible); } + None } @@ -836,8 +842,8 @@ fn mysql_setup_queries_for_database_with_mode( // MySQL defaults group_concat_max_len to 1024, which silently truncates // GROUP_CONCAT results. Skip it for MySQL protocol-compatible databases // such as old StarRocks versions that reject unknown MySQL variables. - if setup_mode.set_group_concat_max_len() { - queries.push("SET @@group_concat_max_len = 1048576".to_string()); + if let Some(query) = setup_mode.group_concat_max_len_query() { + queries.push(query.to_string()); } // StarRocks/Doris expose external storage (Paimon, Hive, ...) through a // catalog. `SET catalog` must run *before* `USE ` (the database @@ -1087,11 +1093,6 @@ fn mysql_error_should_retry_with_text_protocol(error: &str) -> bool { || lower.contains("this command is not supported in the prepared statement protocol yet") } -fn mysql_error_should_retry_without_group_concat_max_len(error: &str) -> bool { - let lower = error.to_ascii_lowercase(); - lower.contains("group_concat_max_len") && (lower.contains("1193") || lower.contains("unknown system variable")) -} - fn ssl_fallback_url(url: &str) -> Option { if mysql_url_requires_ssl(url) { return None; @@ -4480,22 +4481,32 @@ UNIQUE KEY(`tenant_id`, `name``part`) fn mysql_group_concat_setup_error_retries_without_session_variable() { let error = "MySQL connection failed: Server error: `ERROR HY000 (1193): Unknown system variable,stmt:SET @@group_concat_max_len = 1048576'"; - assert!(mysql_error_should_retry_without_group_concat_max_len(error)); assert_eq!( - mysql_setup_mode_retry_without_group_concat(MySqlSetupMode::Standard, error), + mysql_group_concat_setup_fallback_mode(MySqlSetupMode::Standard, error), Some(MySqlSetupMode::Compatible) ); } + #[test] + fn mysql_proxy_parse_tablename_1105_does_not_disable_group_concat() { + let error = "MySQL connection failed: Server error: `ERROR 07000 (1105): SQL操作失败 (operate fail ) :解析表名出错 ( parse tablename error ) '"; + + assert_eq!(mysql_group_concat_setup_fallback_mode(MySqlSetupMode::Standard, error), None); + } + #[test] fn mysql_group_concat_setup_retry_is_narrow() { - assert!(!mysql_error_should_retry_without_group_concat_max_len( - "MySQL connection failed: Server error: `ERROR HY000 (1193): Unknown system variable,stmt:SET @@sql_mode = ANSI'" - )); assert_eq!( - mysql_setup_mode_retry_without_group_concat( - MySqlSetupMode::Compatible, - "Server error: ERROR HY000 (1193): Unknown system variable,stmt:SET @@group_concat_max_len = 1048576" + mysql_group_concat_setup_fallback_mode( + MySqlSetupMode::Standard, + "MySQL connection failed: Server error: `ERROR HY000 (1193): Unknown system variable,stmt:SET @@sql_mode = ANSI'", + ), + None + ); + assert_eq!( + mysql_group_concat_setup_fallback_mode( + MySqlSetupMode::Standard, + "MySQL connection failed: Server error: `ERROR 07000 (1105): SQL操作失败 (operate fail)'", ), None ); @@ -4505,14 +4516,14 @@ UNIQUE KEY(`tenant_id`, `name``part`) fn mysql_setup_queries_select_requested_database_before_session_init() { let queries = mysql_setup_queries("mysql://root:secret@localhost:3306/app?charset=utf8mb4", &[]); - assert_eq!(queries, vec!["USE `app`", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"]); + assert_eq!(queries, vec!["USE `app`", "SET NAMES utf8mb4", "SET SESSION group_concat_max_len = 1048576"]); } #[test] fn mysql_setup_queries_skip_use_when_database_missing() { let queries = mysql_setup_queries("mysql://root:secret@localhost:3306?charset=utf8mb4", &[]); - assert_eq!(queries, vec!["SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"]); + assert_eq!(queries, vec!["SET NAMES utf8mb4", "SET SESSION group_concat_max_len = 1048576"]); } #[test] @@ -4550,7 +4561,7 @@ UNIQUE KEY(`tenant_id`, `name``part`) fn mysql_setup_queries_decode_database_name_from_url() { let queries = mysql_setup_queries("mysql://root:secret@localhost:3306/db%2Fname?charset=utf8mb4", &[]); - assert_eq!(queries, vec!["USE `db/name`", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"]); + assert_eq!(queries, vec!["USE `db/name`", "SET NAMES utf8mb4", "SET SESSION group_concat_max_len = 1048576"]); } #[test] @@ -4561,7 +4572,10 @@ UNIQUE KEY(`tenant_id`, `name``part`) &[], ); - assert_eq!(queries, vec!["USE `app``proxy`", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"]); + assert_eq!( + queries, + vec!["USE `app``proxy`", "SET NAMES utf8mb4", "SET SESSION group_concat_max_len = 1048576"] + ); } #[test] @@ -4745,7 +4759,7 @@ UNIQUE KEY(`tenant_id`, `name``part`) fn mysql_setup_queries_default_to_utf8mb4() { assert_eq!( mysql_setup_queries("mysql://host:3306/db", &[]), - vec!["USE `db`", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"] + vec!["USE `db`", "SET NAMES utf8mb4", "SET SESSION group_concat_max_len = 1048576"] ); } @@ -4753,11 +4767,11 @@ UNIQUE KEY(`tenant_id`, `name``part`) fn mysql_setup_queries_use_safe_custom_charset() { assert_eq!( mysql_setup_queries("mysql://host:3306/db?ssl-mode=preferred&charset=gbk", &[]), - vec!["USE `db`", "SET NAMES gbk", "SET @@group_concat_max_len = 1048576"] + vec!["USE `db`", "SET NAMES gbk", "SET SESSION group_concat_max_len = 1048576"] ); assert_eq!( mysql_setup_queries("mysql://host:3306/db?charset=utf8mb4;DROP TABLE users", &[]), - vec!["USE `db`", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"] + vec!["USE `db`", "SET NAMES utf8mb4", "SET SESSION group_concat_max_len = 1048576"] ); } @@ -4770,7 +4784,7 @@ UNIQUE KEY(`tenant_id`, `name``part`) vec![ "USE `db`", "SET NAMES utf8mb4", - "SET @@group_concat_max_len = 1048576", + "SET SESSION group_concat_max_len = 1048576", "SET ob_query_timeout = 30000000" ] ); @@ -4780,7 +4794,12 @@ UNIQUE KEY(`tenant_id`, `name``part`) fn mysql_setup_queries_apply_explicit_time_zone() { assert_eq!( mysql_setup_queries("mysql://host:3306/db?time_zone=%2B08%3A00&charset=utf8mb4", &[]), - vec!["USE `db`", "SET time_zone = '+08:00'", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"] + vec![ + "USE `db`", + "SET time_zone = '+08:00'", + "SET NAMES utf8mb4", + "SET SESSION group_concat_max_len = 1048576" + ] ); assert_eq!( mysql_setup_queries("mysql://host:3306/db?time-zone=Asia%2FShanghai", &[]), @@ -4788,7 +4807,7 @@ UNIQUE KEY(`tenant_id`, `name``part`) "USE `db`", "SET time_zone = 'Asia/Shanghai'", "SET NAMES utf8mb4", - "SET @@group_concat_max_len = 1048576" + "SET SESSION group_concat_max_len = 1048576" ] ); } @@ -4797,11 +4816,21 @@ UNIQUE KEY(`tenant_id`, `name``part`) fn mysql_setup_queries_apply_jdbc_time_zone_aliases() { assert_eq!( mysql_setup_queries("mysql://host:3306/db?serverTimezone=GMT%2B8", &[]), - vec!["USE `db`", "SET time_zone = '+08:00'", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"] + vec![ + "USE `db`", + "SET time_zone = '+08:00'", + "SET NAMES utf8mb4", + "SET SESSION group_concat_max_len = 1048576" + ] ); assert_eq!( mysql_setup_queries("mysql://host:3306/db?connectionTimeZone=UTC", &[]), - vec!["USE `db`", "SET time_zone = '+00:00'", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"] + vec![ + "USE `db`", + "SET time_zone = '+00:00'", + "SET NAMES utf8mb4", + "SET SESSION group_concat_max_len = 1048576" + ] ); } @@ -4813,12 +4842,17 @@ UNIQUE KEY(`tenant_id`, `name``part`) "USE `db`", "SET time_zone = 'Asia/Shanghai'", "SET NAMES utf8mb4", - "SET @@group_concat_max_len = 1048576" + "SET SESSION group_concat_max_len = 1048576" ] ); assert_eq!( mysql_setup_queries("mysql://host:3306/db?time_zone=%2B08%3A00&loc=UTC", &[]), - vec!["USE `db`", "SET time_zone = '+08:00'", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"] + vec![ + "USE `db`", + "SET time_zone = '+08:00'", + "SET NAMES utf8mb4", + "SET SESSION group_concat_max_len = 1048576" + ] ); } @@ -4826,7 +4860,7 @@ UNIQUE KEY(`tenant_id`, `name``part`) fn mysql_setup_queries_ignore_unsafe_time_zone_values() { assert_eq!( mysql_setup_queries("mysql://host:3306/db?time_zone=%2B08%3A00%27%3BDROP%20TABLE%20users", &[]), - vec!["USE `db`", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"] + vec!["USE `db`", "SET NAMES utf8mb4", "SET SESSION group_concat_max_len = 1048576"] ); } @@ -4839,7 +4873,7 @@ UNIQUE KEY(`tenant_id`, `name``part`) vec![ "USE `clip`", "SET NAMES utf8mb4", - "SET @@group_concat_max_len = 1048576", + "SET SESSION group_concat_max_len = 1048576", "SET catalog = `paimon_catalog`" ] ); @@ -4849,7 +4883,7 @@ UNIQUE KEY(`tenant_id`, `name``part`) fn mysql_setup_queries_switch_catalog_without_database() { assert_eq!( mysql_setup_queries("mysql://host:3306/?catalog=paimon_catalog", &[]), - vec!["SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576", "SET catalog = `paimon_catalog`"] + vec!["SET NAMES utf8mb4", "SET SESSION group_concat_max_len = 1048576", "SET catalog = `paimon_catalog`"] ); } @@ -4857,7 +4891,12 @@ UNIQUE KEY(`tenant_id`, `name``part`) fn mysql_setup_queries_decodes_catalog_parameter() { assert_eq!( mysql_setup_queries("mysql://host:3306/db?catalog=my%5Fcatalog", &[]), - vec!["USE `db`", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576", "SET catalog = `my_catalog`"] + vec![ + "USE `db`", + "SET NAMES utf8mb4", + "SET SESSION group_concat_max_len = 1048576", + "SET catalog = `my_catalog`" + ] ); } @@ -4865,7 +4904,7 @@ UNIQUE KEY(`tenant_id`, `name``part`) fn mysql_setup_queries_omits_catalog_when_absent() { assert_eq!( mysql_setup_queries("mysql://host:3306/db?charset=utf8mb4", &[]), - vec!["USE `db`", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"] + vec!["USE `db`", "SET NAMES utf8mb4", "SET SESSION group_concat_max_len = 1048576"] ); } diff --git a/crates/dbx-core/src/models/connection.rs b/crates/dbx-core/src/models/connection.rs index 19e020d5e2..806b2bd756 100644 --- a/crates/dbx-core/src/models/connection.rs +++ b/crates/dbx-core/src/models/connection.rs @@ -2184,6 +2184,16 @@ mod tests { assert_eq!(config.effective_query_timeout_secs(), 3600); } + #[test] + fn mysql_url_encodes_proxy_style_username_with_colons_and_at() { + let config = mysql_config("1234:xxx@xx.xx:abc123", "p@ss:word", None); + + assert_eq!( + config.connection_url(), + "mysql://1234%3Axxx%40xx%2Exx%3Aabc123:p%40ss%3Aword@10.1.2.3:2883?ssl-mode=disabled&charset=utf8mb4" + ); + } + #[test] fn mysql_url_encodes_oceanbase_username() { let config = mysql_config("user@tenant#cluster", "secret", None);