fix(mysql): preserve group concat metadata limit
This commit is contained in:
1 parent
a9baa8b220
commit
4e0bec13f1
2 files changed
+92
-43
No files matched your search
@@ -598,8 +598,11 @@ enum MySqlSetupMode {
|
||||
}
|
||||
|
||||
impl MySqlSetupMode {
|
||||
fn set_group_concat_max_len(self) -> bool {
|
||||
self == Self::Standard
|
||||
fn group_concat_max_len_query(self) -> Option<&'static str> {
|
||||
match self {
|
||||
Self::Standard => Some("SET SESSION group_concat_max_len = 1048576"),
|
||||
Self::Compatible => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -617,12 +620,10 @@ async fn verify_pool_connection_with_setup_fallback(
|
||||
match verify_pool_connection(&pool, timeout).await {
|
||||
Ok(()) => Ok(pool),
|
||||
Err(err) => {
|
||||
let Some(fallback_mode) = mysql_setup_mode_retry_without_group_concat(setup_mode, &err) else {
|
||||
let Some(fallback_mode) = mysql_group_concat_setup_fallback_mode(setup_mode, &err) else {
|
||||
return Err(err);
|
||||
};
|
||||
log::info!(
|
||||
"MySQL server rejected optional group_concat_max_len setup; retrying without that session setting"
|
||||
);
|
||||
log::info!("MySQL server rejected group_concat_max_len setup syntax; retrying with {fallback_mode:?} mode");
|
||||
let fallback_pool = create_pool(
|
||||
url,
|
||||
ca_cert_path,
|
||||
@@ -637,11 +638,16 @@ async fn verify_pool_connection_with_setup_fallback(
|
||||
}
|
||||
}
|
||||
|
||||
fn mysql_setup_mode_retry_without_group_concat(setup_mode: MySqlSetupMode, error: &str) -> Option<MySqlSetupMode> {
|
||||
// group_concat_max_len improves real MySQL metadata reads, but some MySQL-compatible proxies reject the variable.
|
||||
if setup_mode.set_group_concat_max_len() && mysql_error_should_retry_without_group_concat_max_len(error) {
|
||||
fn mysql_group_concat_setup_fallback_mode(setup_mode: MySqlSetupMode, error: &str) -> Option<MySqlSetupMode> {
|
||||
if setup_mode != MySqlSetupMode::Standard {
|
||||
return None;
|
||||
}
|
||||
|
||||
let lower = error.to_ascii_lowercase();
|
||||
if lower.contains("group_concat_max_len") && (lower.contains("1193") || lower.contains("unknown system variable")) {
|
||||
return Some(MySqlSetupMode::Compatible);
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
@@ -836,8 +842,8 @@ fn mysql_setup_queries_for_database_with_mode(
|
||||
// MySQL defaults group_concat_max_len to 1024, which silently truncates
|
||||
// GROUP_CONCAT results. Skip it for MySQL protocol-compatible databases
|
||||
// such as old StarRocks versions that reject unknown MySQL variables.
|
||||
if setup_mode.set_group_concat_max_len() {
|
||||
queries.push("SET @@group_concat_max_len = 1048576".to_string());
|
||||
if let Some(query) = setup_mode.group_concat_max_len_query() {
|
||||
queries.push(query.to_string());
|
||||
}
|
||||
// StarRocks/Doris expose external storage (Paimon, Hive, ...) through a
|
||||
// catalog. `SET catalog` must run *before* `USE <database>` (the database
|
||||
@@ -1087,11 +1093,6 @@ fn mysql_error_should_retry_with_text_protocol(error: &str) -> bool {
|
||||
|| lower.contains("this command is not supported in the prepared statement protocol yet")
|
||||
}
|
||||
|
||||
fn mysql_error_should_retry_without_group_concat_max_len(error: &str) -> bool {
|
||||
let lower = error.to_ascii_lowercase();
|
||||
lower.contains("group_concat_max_len") && (lower.contains("1193") || lower.contains("unknown system variable"))
|
||||
}
|
||||
|
||||
fn ssl_fallback_url(url: &str) -> Option<String> {
|
||||
if mysql_url_requires_ssl(url) {
|
||||
return None;
|
||||
@@ -4480,22 +4481,32 @@ UNIQUE KEY(`tenant_id`, `name``part`)
|
||||
fn mysql_group_concat_setup_error_retries_without_session_variable() {
|
||||
let error = "MySQL connection failed: Server error: `ERROR HY000 (1193): Unknown system variable,stmt:SET @@group_concat_max_len = 1048576'";
|
||||
|
||||
assert!(mysql_error_should_retry_without_group_concat_max_len(error));
|
||||
assert_eq!(
|
||||
mysql_setup_mode_retry_without_group_concat(MySqlSetupMode::Standard, error),
|
||||
mysql_group_concat_setup_fallback_mode(MySqlSetupMode::Standard, error),
|
||||
Some(MySqlSetupMode::Compatible)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_proxy_parse_tablename_1105_does_not_disable_group_concat() {
|
||||
let error = "MySQL connection failed: Server error: `ERROR 07000 (1105): SQL操作失败 (operate fail ) :解析表名出错 ( parse tablename error ) '";
|
||||
|
||||
assert_eq!(mysql_group_concat_setup_fallback_mode(MySqlSetupMode::Standard, error), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_group_concat_setup_retry_is_narrow() {
|
||||
assert!(!mysql_error_should_retry_without_group_concat_max_len(
|
||||
"MySQL connection failed: Server error: `ERROR HY000 (1193): Unknown system variable,stmt:SET @@sql_mode = ANSI'"
|
||||
));
|
||||
assert_eq!(
|
||||
mysql_setup_mode_retry_without_group_concat(
|
||||
MySqlSetupMode::Compatible,
|
||||
"Server error: ERROR HY000 (1193): Unknown system variable,stmt:SET @@group_concat_max_len = 1048576"
|
||||
mysql_group_concat_setup_fallback_mode(
|
||||
MySqlSetupMode::Standard,
|
||||
"MySQL connection failed: Server error: `ERROR HY000 (1193): Unknown system variable,stmt:SET @@sql_mode = ANSI'",
|
||||
),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
mysql_group_concat_setup_fallback_mode(
|
||||
MySqlSetupMode::Standard,
|
||||
"MySQL connection failed: Server error: `ERROR 07000 (1105): SQL操作失败 (operate fail)'",
|
||||
),
|
||||
None
|
||||
);
|
||||
@@ -4505,14 +4516,14 @@ UNIQUE KEY(`tenant_id`, `name``part`)
|
||||
fn mysql_setup_queries_select_requested_database_before_session_init() {
|
||||
let queries = mysql_setup_queries("mysql://root:secret@localhost:3306/app?charset=utf8mb4", &[]);
|
||||
|
||||
assert_eq!(queries, vec!["USE `app`", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"]);
|
||||
assert_eq!(queries, vec!["USE `app`", "SET NAMES utf8mb4", "SET SESSION group_concat_max_len = 1048576"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_setup_queries_skip_use_when_database_missing() {
|
||||
let queries = mysql_setup_queries("mysql://root:secret@localhost:3306?charset=utf8mb4", &[]);
|
||||
|
||||
assert_eq!(queries, vec!["SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"]);
|
||||
assert_eq!(queries, vec!["SET NAMES utf8mb4", "SET SESSION group_concat_max_len = 1048576"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4550,7 +4561,7 @@ UNIQUE KEY(`tenant_id`, `name``part`)
|
||||
fn mysql_setup_queries_decode_database_name_from_url() {
|
||||
let queries = mysql_setup_queries("mysql://root:secret@localhost:3306/db%2Fname?charset=utf8mb4", &[]);
|
||||
|
||||
assert_eq!(queries, vec!["USE `db/name`", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"]);
|
||||
assert_eq!(queries, vec!["USE `db/name`", "SET NAMES utf8mb4", "SET SESSION group_concat_max_len = 1048576"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4561,7 +4572,10 @@ UNIQUE KEY(`tenant_id`, `name``part`)
|
||||
&[],
|
||||
);
|
||||
|
||||
assert_eq!(queries, vec!["USE `app``proxy`", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"]);
|
||||
assert_eq!(
|
||||
queries,
|
||||
vec!["USE `app``proxy`", "SET NAMES utf8mb4", "SET SESSION group_concat_max_len = 1048576"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4745,7 +4759,7 @@ UNIQUE KEY(`tenant_id`, `name``part`)
|
||||
fn mysql_setup_queries_default_to_utf8mb4() {
|
||||
assert_eq!(
|
||||
mysql_setup_queries("mysql://host:3306/db", &[]),
|
||||
vec!["USE `db`", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"]
|
||||
vec!["USE `db`", "SET NAMES utf8mb4", "SET SESSION group_concat_max_len = 1048576"]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4753,11 +4767,11 @@ UNIQUE KEY(`tenant_id`, `name``part`)
|
||||
fn mysql_setup_queries_use_safe_custom_charset() {
|
||||
assert_eq!(
|
||||
mysql_setup_queries("mysql://host:3306/db?ssl-mode=preferred&charset=gbk", &[]),
|
||||
vec!["USE `db`", "SET NAMES gbk", "SET @@group_concat_max_len = 1048576"]
|
||||
vec!["USE `db`", "SET NAMES gbk", "SET SESSION group_concat_max_len = 1048576"]
|
||||
);
|
||||
assert_eq!(
|
||||
mysql_setup_queries("mysql://host:3306/db?charset=utf8mb4;DROP TABLE users", &[]),
|
||||
vec!["USE `db`", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"]
|
||||
vec!["USE `db`", "SET NAMES utf8mb4", "SET SESSION group_concat_max_len = 1048576"]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4770,7 +4784,7 @@ UNIQUE KEY(`tenant_id`, `name``part`)
|
||||
vec![
|
||||
"USE `db`",
|
||||
"SET NAMES utf8mb4",
|
||||
"SET @@group_concat_max_len = 1048576",
|
||||
"SET SESSION group_concat_max_len = 1048576",
|
||||
"SET ob_query_timeout = 30000000"
|
||||
]
|
||||
);
|
||||
@@ -4780,7 +4794,12 @@ UNIQUE KEY(`tenant_id`, `name``part`)
|
||||
fn mysql_setup_queries_apply_explicit_time_zone() {
|
||||
assert_eq!(
|
||||
mysql_setup_queries("mysql://host:3306/db?time_zone=%2B08%3A00&charset=utf8mb4", &[]),
|
||||
vec!["USE `db`", "SET time_zone = '+08:00'", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"]
|
||||
vec![
|
||||
"USE `db`",
|
||||
"SET time_zone = '+08:00'",
|
||||
"SET NAMES utf8mb4",
|
||||
"SET SESSION group_concat_max_len = 1048576"
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
mysql_setup_queries("mysql://host:3306/db?time-zone=Asia%2FShanghai", &[]),
|
||||
@@ -4788,7 +4807,7 @@ UNIQUE KEY(`tenant_id`, `name``part`)
|
||||
"USE `db`",
|
||||
"SET time_zone = 'Asia/Shanghai'",
|
||||
"SET NAMES utf8mb4",
|
||||
"SET @@group_concat_max_len = 1048576"
|
||||
"SET SESSION group_concat_max_len = 1048576"
|
||||
]
|
||||
);
|
||||
}
|
||||
@@ -4797,11 +4816,21 @@ UNIQUE KEY(`tenant_id`, `name``part`)
|
||||
fn mysql_setup_queries_apply_jdbc_time_zone_aliases() {
|
||||
assert_eq!(
|
||||
mysql_setup_queries("mysql://host:3306/db?serverTimezone=GMT%2B8", &[]),
|
||||
vec!["USE `db`", "SET time_zone = '+08:00'", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"]
|
||||
vec![
|
||||
"USE `db`",
|
||||
"SET time_zone = '+08:00'",
|
||||
"SET NAMES utf8mb4",
|
||||
"SET SESSION group_concat_max_len = 1048576"
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
mysql_setup_queries("mysql://host:3306/db?connectionTimeZone=UTC", &[]),
|
||||
vec!["USE `db`", "SET time_zone = '+00:00'", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"]
|
||||
vec![
|
||||
"USE `db`",
|
||||
"SET time_zone = '+00:00'",
|
||||
"SET NAMES utf8mb4",
|
||||
"SET SESSION group_concat_max_len = 1048576"
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4813,12 +4842,17 @@ UNIQUE KEY(`tenant_id`, `name``part`)
|
||||
"USE `db`",
|
||||
"SET time_zone = 'Asia/Shanghai'",
|
||||
"SET NAMES utf8mb4",
|
||||
"SET @@group_concat_max_len = 1048576"
|
||||
"SET SESSION group_concat_max_len = 1048576"
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
mysql_setup_queries("mysql://host:3306/db?time_zone=%2B08%3A00&loc=UTC", &[]),
|
||||
vec!["USE `db`", "SET time_zone = '+08:00'", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"]
|
||||
vec![
|
||||
"USE `db`",
|
||||
"SET time_zone = '+08:00'",
|
||||
"SET NAMES utf8mb4",
|
||||
"SET SESSION group_concat_max_len = 1048576"
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4826,7 +4860,7 @@ UNIQUE KEY(`tenant_id`, `name``part`)
|
||||
fn mysql_setup_queries_ignore_unsafe_time_zone_values() {
|
||||
assert_eq!(
|
||||
mysql_setup_queries("mysql://host:3306/db?time_zone=%2B08%3A00%27%3BDROP%20TABLE%20users", &[]),
|
||||
vec!["USE `db`", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"]
|
||||
vec!["USE `db`", "SET NAMES utf8mb4", "SET SESSION group_concat_max_len = 1048576"]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4839,7 +4873,7 @@ UNIQUE KEY(`tenant_id`, `name``part`)
|
||||
vec![
|
||||
"USE `clip`",
|
||||
"SET NAMES utf8mb4",
|
||||
"SET @@group_concat_max_len = 1048576",
|
||||
"SET SESSION group_concat_max_len = 1048576",
|
||||
"SET catalog = `paimon_catalog`"
|
||||
]
|
||||
);
|
||||
@@ -4849,7 +4883,7 @@ UNIQUE KEY(`tenant_id`, `name``part`)
|
||||
fn mysql_setup_queries_switch_catalog_without_database() {
|
||||
assert_eq!(
|
||||
mysql_setup_queries("mysql://host:3306/?catalog=paimon_catalog", &[]),
|
||||
vec!["SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576", "SET catalog = `paimon_catalog`"]
|
||||
vec!["SET NAMES utf8mb4", "SET SESSION group_concat_max_len = 1048576", "SET catalog = `paimon_catalog`"]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4857,7 +4891,12 @@ UNIQUE KEY(`tenant_id`, `name``part`)
|
||||
fn mysql_setup_queries_decodes_catalog_parameter() {
|
||||
assert_eq!(
|
||||
mysql_setup_queries("mysql://host:3306/db?catalog=my%5Fcatalog", &[]),
|
||||
vec!["USE `db`", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576", "SET catalog = `my_catalog`"]
|
||||
vec![
|
||||
"USE `db`",
|
||||
"SET NAMES utf8mb4",
|
||||
"SET SESSION group_concat_max_len = 1048576",
|
||||
"SET catalog = `my_catalog`"
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4865,7 +4904,7 @@ UNIQUE KEY(`tenant_id`, `name``part`)
|
||||
fn mysql_setup_queries_omits_catalog_when_absent() {
|
||||
assert_eq!(
|
||||
mysql_setup_queries("mysql://host:3306/db?charset=utf8mb4", &[]),
|
||||
vec!["USE `db`", "SET NAMES utf8mb4", "SET @@group_concat_max_len = 1048576"]
|
||||
vec!["USE `db`", "SET NAMES utf8mb4", "SET SESSION group_concat_max_len = 1048576"]
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -2184,6 +2184,16 @@ mod tests {
|
||||
assert_eq!(config.effective_query_timeout_secs(), 3600);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_url_encodes_proxy_style_username_with_colons_and_at() {
|
||||
let config = mysql_config("1234:xxx@xx.xx:abc123", "p@ss:word", None);
|
||||
|
||||
assert_eq!(
|
||||
config.connection_url(),
|
||||
"mysql://1234%3Axxx%40xx%2Exx%3Aabc123:p%40ss%3Aword@10.1.2.3:2883?ssl-mode=disabled&charset=utf8mb4"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_url_encodes_oceanbase_username() {
|
||||
let config = mysql_config("user@tenant#cluster", "secret", None);
|
||||
|
||||
Reference in new issue
Block a user