fix(mysql): optimize stored procedure loading

This commit is contained in:
zipg authored and GitHub committed 2026-07-11 00:32:49 +08:00
1 parent 16d7aef4db
commit 3a5ef776b1
6 files changed
+424 -112

No files matched your search

+146 -38
View File
@@ -1957,7 +1957,29 @@ pub async fn list_tables_show(pool: &MySqlPool, database: &str) -> Result<Vec<Ta
list_tables_show_with_status(pool, database).await.map(|(tables, _)| tables)
}
fn list_tables_objects_sql(database: &str) -> String {
fn requested_object_type(object_types: Option<&[String]>, object_type: &str) -> bool {
object_types.is_none_or(|types| {
types.is_empty() || types.iter().any(|candidate| candidate.eq_ignore_ascii_case(object_type))
})
}
fn sql_pagination(limit: Option<usize>, offset: Option<usize>) -> String {
limit.map_or_else(String::new, |limit| format!(" LIMIT {limit} OFFSET {}", offset.unwrap_or(0)))
}
fn list_tables_objects_sql(
database: &str,
object_types: Option<&[String]>,
limit: Option<usize>,
offset: Option<usize>,
) -> String {
let wants_tables = requested_object_type(object_types, "TABLE");
let wants_views = requested_object_type(object_types, "VIEW");
let type_filter = match (wants_tables, wants_views) {
(true, false) => " AND TABLE_TYPE <> 'VIEW'",
(false, true) => " AND TABLE_TYPE = 'VIEW'",
_ => "",
};
format!(
"SELECT TABLE_NAME AS object_name, \
CASE WHEN TABLE_TYPE = 'VIEW' THEN 'VIEW' ELSE 'TABLE' END AS object_type, \
@@ -1967,22 +1989,37 @@ fn list_tables_objects_sql(database: &str) -> String {
NULL AS parent_schema, NULL AS parent_name, \
CASE WHEN TABLE_TYPE = 'VIEW' THEN 1 ELSE 0 END AS sort_order \
FROM information_schema.TABLES \
WHERE TABLE_SCHEMA = {db} \
ORDER BY sort_order, object_name",
WHERE TABLE_SCHEMA = {db}{type_filter} \
ORDER BY sort_order, object_name{pagination}",
db = quote_value(database),
pagination = sql_pagination(limit, offset),
)
}
fn list_routines_sql(database: &str) -> String {
fn list_routines_sql(
database: &str,
object_types: Option<&[String]>,
limit: Option<usize>,
offset: Option<usize>,
) -> String {
let routine_types = [
("PROCEDURE", requested_object_type(object_types, "PROCEDURE")),
("FUNCTION", requested_object_type(object_types, "FUNCTION")),
]
.into_iter()
.filter_map(|(routine_type, requested)| requested.then_some(format!("'{}'", routine_type)))
.collect::<Vec<_>>()
.join(", ");
format!(
"SELECT ROUTINE_NAME AS object_name, ROUTINE_TYPE AS object_type, NULL AS object_comment, \
NULL AS created_at, NULL AS updated_at, \
NULL AS parent_schema, NULL AS parent_name, \
CASE WHEN ROUTINE_TYPE = 'PROCEDURE' THEN 2 ELSE 3 END AS sort_order \
FROM information_schema.ROUTINES \
WHERE ROUTINE_SCHEMA = {db} AND ROUTINE_TYPE IN ('PROCEDURE', 'FUNCTION') \
ORDER BY sort_order, object_name",
WHERE ROUTINE_SCHEMA = {db} AND ROUTINE_TYPE IN ({routine_types}) \
ORDER BY sort_order, object_name{pagination}",
db = quote_value(database),
pagination = sql_pagination(limit, offset),
)
}
@@ -2015,47 +2052,91 @@ fn row_to_object(row: &mysql_async::Row, database: &str) -> ObjectInfo {
}
}
pub async fn list_objects(pool: &MySqlPool, database: &str) -> Result<Vec<ObjectInfo>, String> {
let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?;
pub struct PagedObjectList {
pub objects: Vec<ObjectInfo>,
pub paging_applied: bool,
}
let tables_sql = list_tables_objects_sql(database);
let result = match conn.query_iter(&tables_sql).await {
Ok(result) => result,
Err(err) => {
log::debug!(
"Falling back to SHOW TABLES for object browser database `{database}` after information_schema.TABLES failed: {err}"
);
return list_table_objects_show(pool, database).await;
}
fn object_query_supports_paging(object_types: Option<&[String]>) -> bool {
let Some(object_types) = object_types.filter(|types| !types.is_empty()) else {
return false;
};
let table_rows: Vec<mysql_async::Row> = result.collect_and_drop().await.map_err(|e| e.to_string())?;
if table_rows.is_empty() {
log::debug!(
"Falling back to SHOW TABLES for object browser database `{database}` after information_schema.TABLES returned no named tables"
);
return list_table_objects_show(pool, database).await;
let uses_table_source = object_types
.iter()
.any(|object_type| object_type.eq_ignore_ascii_case("TABLE") || object_type.eq_ignore_ascii_case("VIEW"));
let uses_routine_source = object_types.iter().any(|object_type| {
object_type.eq_ignore_ascii_case("PROCEDURE") || object_type.eq_ignore_ascii_case("FUNCTION")
});
let all_types_supported = object_types.iter().all(|object_type| {
object_type.eq_ignore_ascii_case("TABLE")
|| object_type.eq_ignore_ascii_case("VIEW")
|| object_type.eq_ignore_ascii_case("PROCEDURE")
|| object_type.eq_ignore_ascii_case("FUNCTION")
});
all_types_supported && uses_table_source != uses_routine_source
}
pub async fn list_objects(
pool: &MySqlPool,
database: &str,
object_types: Option<&[String]>,
limit: Option<usize>,
offset: Option<usize>,
) -> Result<PagedObjectList, String> {
let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?;
let wants_tables = requested_object_type(object_types, "TABLE") || requested_object_type(object_types, "VIEW");
let wants_routines =
requested_object_type(object_types, "PROCEDURE") || requested_object_type(object_types, "FUNCTION");
let paging_applied = limit.is_some() && object_query_supports_paging(object_types);
let (query_limit, query_offset) = if paging_applied { (limit, offset) } else { (None, None) };
let mut objects = Vec::new();
if wants_tables {
let tables_sql = list_tables_objects_sql(database, object_types, query_limit, query_offset);
let result = match conn.query_iter(&tables_sql).await {
Ok(result) => result,
Err(err) => {
log::debug!(
"Falling back to SHOW TABLES for object browser database `{database}` after information_schema.TABLES failed: {err}"
);
return list_table_objects_show(pool, database)
.await
.map(|objects| PagedObjectList { objects, paging_applied: false });
}
};
let table_rows: Vec<mysql_async::Row> = result.collect_and_drop().await.map_err(|e| e.to_string())?;
if table_rows.is_empty() && !wants_routines {
log::debug!(
"Falling back to SHOW TABLES for object browser database `{database}` after information_schema.TABLES returned no named tables"
);
return list_table_objects_show(pool, database)
.await
.map(|objects| PagedObjectList { objects, paging_applied: false });
}
objects.extend(table_rows.iter().map(|row| row_to_object(row, database)));
}
let mut objects: Vec<ObjectInfo> = table_rows.iter().map(|row| row_to_object(row, database)).collect();
// Routines are queried separately: some MySQL-compatible servers (sharding proxies,
// OceanBase/TiDB variants, restricted accounts) reject information_schema.ROUTINES with
// ER_UNKNOWN_ERROR (1105). Degrading gracefully keeps tables/views usable.
let routines_sql = list_routines_sql(database);
match conn.query_iter(&routines_sql).await {
Ok(result) => match result.collect_and_drop::<mysql_async::Row>().await {
Ok(routine_rows) => {
objects.extend(routine_rows.iter().map(|row| row_to_object(row, database)));
}
if wants_routines {
let routines_sql = list_routines_sql(database, object_types, query_limit, query_offset);
match conn.query_iter(&routines_sql).await {
Ok(result) => match result.collect_and_drop::<mysql_async::Row>().await {
Ok(routine_rows) => {
objects.extend(routine_rows.iter().map(|row| row_to_object(row, database)));
}
Err(e) => {
log::warn!("Skipping routines for database `{}` in object browser: {}", database, e);
}
},
Err(e) => {
log::warn!("Skipping routines for database `{}` in object browser: {}", database, e);
}
},
Err(e) => {
log::warn!("Skipping routines for database `{}` in object browser: {}", database, e);
}
}
Ok(objects)
Ok(PagedObjectList { objects, paging_applied })
}
pub async fn list_object_statistics(pool: &MySqlPool, database: &str) -> Result<Vec<ObjectStatistics>, String> {
@@ -2115,7 +2196,7 @@ pub async fn list_table_objects_show(pool: &MySqlPool, database: &str) -> Result
async fn list_routine_objects(pool: &MySqlPool, database: &str) -> Result<Vec<ObjectInfo>, String> {
let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?;
let routines_sql = list_routines_sql(database);
let routines_sql = list_routines_sql(database, None, None, None);
let result = conn.query_iter(&routines_sql).await.map_err(|e| e.to_string())?;
let rows: Vec<mysql_async::Row> = result.collect_and_drop().await.map_err(|e| e.to_string())?;
Ok(rows.iter().map(|row| row_to_object(row, database)).collect())
@@ -2125,7 +2206,7 @@ pub async fn list_completion_objects(pool: &MySqlPool, database: &str) -> Result
let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?;
let mut objects = Vec::new();
let routines_sql = list_routines_sql(database);
let routines_sql = list_routines_sql(database, None, None, None);
match conn.query_iter(&routines_sql).await {
Ok(result) => match result.collect_and_drop::<mysql_async::Row>().await {
Ok(rows) => objects.extend(rows.iter().map(|row| row_to_object(row, database))),
@@ -3797,7 +3878,7 @@ mod tests {
#[test]
fn mysql_list_tables_objects_sql_includes_timestamps() {
let sql = list_tables_objects_sql("app");
let sql = list_tables_objects_sql("app", None, None, None);
assert!(sql.contains("information_schema.TABLES"));
assert!(!sql.contains("information_schema.ROUTINES"));
@@ -3939,7 +4020,7 @@ mod tests {
#[test]
fn mysql_list_routines_sql_is_independent_of_tables() {
let sql = list_routines_sql("app");
let sql = list_routines_sql("app", None, None, None);
assert!(sql.contains("information_schema.ROUTINES"));
assert!(!sql.contains("information_schema.TABLES"));
@@ -3950,6 +4031,33 @@ mod tests {
assert!(!sql.contains("CREATED AS created_at"));
}
#[test]
fn mysql_list_routines_sql_honors_requested_type_and_paging() {
let object_types = vec!["PROCEDURE".to_string()];
let sql = list_routines_sql("app", Some(&object_types), Some(101), Some(200));
assert!(sql.contains("ROUTINE_TYPE IN ('PROCEDURE')"));
assert!(!sql.contains("'FUNCTION'"));
assert!(sql.ends_with("LIMIT 101 OFFSET 200"));
}
#[test]
fn mysql_list_tables_objects_sql_honors_requested_type_and_paging() {
let object_types = vec!["VIEW".to_string()];
let sql = list_tables_objects_sql("app", Some(&object_types), Some(51), Some(100));
assert!(sql.contains("TABLE_TYPE = 'VIEW'"));
assert!(sql.ends_with("LIMIT 51 OFFSET 100"));
}
#[test]
fn mysql_object_query_only_pages_within_one_metadata_source() {
assert!(object_query_supports_paging(Some(&["PROCEDURE".to_string()])));
assert!(object_query_supports_paging(Some(&["TABLE".to_string(), "VIEW".to_string()])));
assert!(!object_query_supports_paging(Some(&["TABLE".to_string(), "PROCEDURE".to_string()])));
assert!(!object_query_supports_paging(None));
}
#[test]
fn mysql_completion_triggers_sql_lists_database_triggers() {
let sql = list_completion_triggers_sql("app");
+70 -39
View File
@@ -3216,14 +3216,15 @@ pub async fn list_objects_core(
retry_metadata_connection(state, connection_id, Some(database), || async {
let objects = list_objects_once(state, connection_id, database, schema, filter, limit, offset, object_types)
.await
.map(|objects| {
let final_offset = if oracle_agent_paging_likely_applied(use_oracle_agent_paging, limit, objects.len())
.map(|outcome| {
let final_offset = if outcome.paging_applied
|| oracle_agent_paging_likely_applied(use_oracle_agent_paging, limit, outcome.objects.len())
{
Some(0)
} else {
offset
};
filter_object_infos(objects, filter, limit, final_offset, object_types)
filter_object_infos(outcome.objects, filter, limit, final_offset, object_types)
})?;
Ok(objects)
})
@@ -3534,6 +3535,15 @@ async fn list_object_statistics_once(
}
}
struct ObjectListOutcome {
objects: Vec<db::ObjectInfo>,
paging_applied: bool,
}
fn unpaged_object_list(objects: Vec<db::ObjectInfo>) -> ObjectListOutcome {
ObjectListOutcome { objects, paging_applied: false }
}
async fn list_objects_once(
state: &AppState,
connection_id: &str,
@@ -3543,9 +3553,11 @@ async fn list_objects_once(
limit: Option<usize>,
offset: Option<usize>,
object_types: Option<&[String]>,
) -> Result<Vec<db::ObjectInfo>, String> {
) -> Result<ObjectListOutcome, String> {
let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?;
let db_config = connection_config(state, connection_id).await;
let (mysql_limit, mysql_offset) =
if filter.is_none_or(|value| value.trim().is_empty()) { (limit, offset) } else { (None, None) };
{
let connections = state.connections.read().await;
@@ -3553,7 +3565,7 @@ async fn list_objects_once(
if let Some(ext_pool) = extract_pool!(&connections, &pool_key, ExternalTabular) {
drop(connections);
let cache = ext_pool.cache.clone();
return tokio::task::spawn_blocking(move || {
let objects = tokio::task::spawn_blocking(move || {
let con = cache.lock().map_err(|e| e.to_string())?;
Ok(duckdb_query_tables(&con)?
.into_iter()
@@ -3572,6 +3584,7 @@ async fn list_objects_once(
})
.await
.map_err(|e| e.to_string())?;
return objects.map(unpaged_object_list);
}
if let Some(PoolKind::ExternalDriver { config, session, .. }) = connections.get(&pool_key) {
let config = config.clone();
@@ -3586,7 +3599,8 @@ async fn list_objects_once(
filter,
object_types,
)
.await;
.await
.map(unpaged_object_list);
}
let mut params =
serde_json::json!({ "connection": config.as_ref(), "database": database, "schema": schema });
@@ -3602,9 +3616,14 @@ async fn list_objects_once(
params,
agent_metadata_timeout(Some(config.as_ref())),
)
.await;
.await
.map(unpaged_object_list);
}
if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) {
drop(connections);
let mut client = client.lock().await;
return db::sqlserver::list_objects(&mut client, schema).await.map(unpaged_object_list);
}
try_sqlserver!(connections, &pool_key, list_objects, schema);
if let Some(client) = extract_pool!(&connections, &pool_key, Agent) {
let is_oracle = db_config.as_ref().is_some_and(|config| config.db_type == DatabaseType::Oracle);
let use_oracle_agent_paging = db_config.as_ref().is_some_and(is_default_oracle_agent_config);
@@ -3614,7 +3633,9 @@ async fn list_objects_once(
let fallback_config = db_config.clone();
drop(connections);
if is_oracle && !use_oracle_agent_paging {
return oracle_agent_list_objects(client, database, schema, timeout_duration).await;
return oracle_agent_list_objects(client, database, schema, timeout_duration)
.await
.map(unpaged_object_list);
}
let mut client = client.lock().await;
let agent_filter = if filter_locally_after_oracle_comments { None } else { filter };
@@ -3655,13 +3676,15 @@ async fn list_objects_once(
)
.await?;
}
return Ok(objects);
return Ok(unpaged_object_list(objects));
}
Ok(objects) => {
if let Some(config) = fallback_config.as_ref() {
match native_postgres_metadata_pool(state, connection_id, database, config).await {
Ok(Some(pool)) => return db::postgres::list_objects(&pool, schema).await,
Ok(None) => return Ok(objects),
Ok(Some(pool)) => {
return db::postgres::list_objects(&pool, schema).await.map(unpaged_object_list)
}
Ok(None) => return Ok(unpaged_object_list(objects)),
Err(error) => {
log::warn!(
"[schema][agent:list_objects:fallback-failed] connection_id={} database={} schema={} error={}",
@@ -3673,16 +3696,20 @@ async fn list_objects_once(
}
}
}
return Ok(objects);
return Ok(unpaged_object_list(objects));
}
Err(agent_error) => {
if let Some(config) = fallback_config.as_ref() {
if let Some(pool) =
native_postgres_metadata_pool(state, connection_id, database, config).await?
{
return db::postgres::list_objects(&pool, schema).await.map_err(|fallback_error| {
format!("{agent_error}\n\nNative PostgreSQL metadata fallback failed: {fallback_error}")
});
return db::postgres::list_objects(&pool, schema).await.map(unpaged_object_list).map_err(
|fallback_error| {
format!(
"{agent_error}\n\nNative PostgreSQL metadata fallback failed: {fallback_error}"
)
},
);
}
}
return Err(agent_error);
@@ -3698,36 +3725,40 @@ async fn list_objects_once(
PoolKind::Mysql(p, mode) => {
// Note: mysql and ob_oracle take different second args (database vs schema)
if *mode == MysqlMode::OceanBaseOracle {
db::ob_oracle::list_objects(p, schema).await
db::ob_oracle::list_objects(p, schema).await.map(unpaged_object_list)
} else if db_config.as_ref().is_some_and(is_manticoresearch_config) {
db::manticoresearch::list_objects(p, database).await
db::manticoresearch::list_objects(p, database).await.map(unpaged_object_list)
} else if db_config.as_ref().is_some_and(is_doris_family_config) {
db::mysql::list_table_objects_show(p, database).await
db::mysql::list_table_objects_show(p, database).await.map(unpaged_object_list)
} else {
db::mysql::list_objects(p, database).await
db::mysql::list_objects(p, database, object_types, mysql_limit, mysql_offset)
.await
.map(|result| ObjectListOutcome { objects: result.objects, paging_applied: result.paging_applied })
}
}
PoolKind::Postgres(p) if db_config.as_ref().is_some_and(is_questdb_config) => {
db::questdb::list_objects(p, schema).await
db::questdb::list_objects(p, schema).await.map(unpaged_object_list)
}
PoolKind::Postgres(p) => db::postgres::list_objects(p, schema).await,
PoolKind::Postgres(p) => db::postgres::list_objects(p, schema).await.map(unpaged_object_list),
_ => {
drop(connections);
Ok(list_tables_core(state, connection_id, database, schema, None, None, None, None)
.await?
.into_iter()
.map(|table| db::ObjectInfo {
name: table.name,
object_type: table.table_type,
schema: if schema.is_empty() { None } else { Some(schema.to_string()) },
signature: None,
comment: table.comment,
created_at: None,
updated_at: None,
parent_schema: table.parent_schema,
parent_name: table.parent_name,
})
.collect())
Ok(unpaged_object_list(
list_tables_core(state, connection_id, database, schema, None, None, None, None)
.await?
.into_iter()
.map(|table| db::ObjectInfo {
name: table.name,
object_type: table.table_type,
schema: if schema.is_empty() { None } else { Some(schema.to_string()) },
signature: None,
comment: table.comment,
created_at: None,
updated_at: None,
parent_schema: table.parent_schema,
parent_name: table.parent_name,
})
.collect(),
))
}
}
}
@@ -3826,8 +3857,8 @@ async fn list_completion_objects_once(
PoolKind::Postgres(p) => db::postgres::list_objects(p, schema).await.map(filter_completion_objects),
PoolKind::SqlServer(_) => {
drop(connections);
let objects = list_objects_once(state, connection_id, database, schema, None, None, None, None).await?;
Ok(filter_completion_objects(objects))
let outcome = list_objects_once(state, connection_id, database, schema, None, None, None, None).await?;
Ok(filter_completion_objects(outcome.objects))
}
_ => Ok(Vec::new()),
}
@@ -82,7 +82,9 @@ pub(in crate::schema) async fn list_objects(
PoolKind::Mysql(p, _) if config.is_some_and(is_doris_family_config) => {
db::mysql::list_table_objects_show(p, database).await.map(Some)
}
PoolKind::Mysql(p, _) => db::mysql::list_objects(p, database).await.map(Some),
PoolKind::Mysql(p, _) => db::mysql::list_objects(p, database, None, None, None)
.await
.map(|result| Some(result.objects)),
PoolKind::Postgres(p) if config.is_some_and(is_questdb_config) => {
db::questdb::list_objects(p, schema).await.map(Some)
}