fix(presto): optimize metadata completion

This commit is contained in:
luoianun authored and GitHub committed 2026-07-01 19:05:45 +08:00
1 parent f5f6bc0b43
commit 95e3cb6571
9 files changed
+528 -37

No files matched your search

+219 -14
View File
@@ -1414,9 +1414,17 @@ async fn list_tables_once(
let session = session.clone();
drop(connections);
if uses_presto_like_information_schema_tables(&config.db_type) {
return external_driver_presto_like_tables(session, config.as_ref(), database, schema)
.await
.map(|tables| filter_table_infos(tables, filter, limit, offset, object_types));
return external_driver_presto_like_tables(
session,
config.as_ref(),
database,
schema,
filter,
limit,
offset,
)
.await
.map(|tables| filter_table_infos(tables, filter, limit, offset, object_types));
}
return session
.invoke_with_timeout::<Vec<db::TableInfo>>(
@@ -1677,7 +1685,11 @@ async fn external_driver_presto_like_tables(
config: &ConnectionConfig,
database: &str,
schema: &str,
filter: Option<&str>,
limit: Option<usize>,
offset: Option<usize>,
) -> Result<Vec<db::TableInfo>, String> {
let query_limit = limit.map(|limit| limit.saturating_add(offset.unwrap_or(0)).max(1)).unwrap_or(100000);
let result: db::QueryResult = session
.invoke_with_timeout(
"executeQuery",
@@ -1685,8 +1697,8 @@ async fn external_driver_presto_like_tables(
"connection": config,
"database": database,
"schema": schema,
"sql": presto_like_information_schema_tables_sql(database, schema),
"maxRows": 100000,
"sql": presto_like_information_schema_tables_sql(database, schema, filter, Some(query_limit)),
"maxRows": query_limit,
"fetchSize": 1000,
"timeoutSecs": 60
}),
@@ -1702,7 +1714,7 @@ async fn external_driver_presto_like_objects(
database: &str,
schema: &str,
) -> Result<Vec<db::ObjectInfo>, String> {
let tables = external_driver_presto_like_tables(session, config, database, schema).await?;
let tables = external_driver_presto_like_tables(session, config, database, schema, None, None, None).await?;
Ok(tables
.into_iter()
.map(|table| db::ObjectInfo {
@@ -1719,18 +1731,72 @@ async fn external_driver_presto_like_objects(
.collect())
}
fn presto_like_information_schema_tables_sql(database: &str, schema: &str) -> String {
async fn external_driver_presto_like_columns(
session: Arc<crate::plugins::PluginDriverSession>,
config: &ConnectionConfig,
database: &str,
schema: &str,
table: &str,
) -> Result<Vec<db::ColumnInfo>, String> {
let result: db::QueryResult = session
.invoke(
"executeQuery",
serde_json::json!({
"connection": config,
"database": database,
"schema": schema,
"sql": presto_like_information_schema_columns_sql(database, schema, table),
"maxRows": 10000,
"fetchSize": 1000,
"timeoutSecs": 60
}),
)
.await?;
Ok(presto_like_columns_from_query_result(&result))
}
fn presto_like_information_schema_tables_sql(
database: &str,
schema: &str,
filter: Option<&str>,
limit: Option<usize>,
) -> String {
let source = if database.trim().is_empty() {
"information_schema.tables".to_string()
} else {
format!("{}.information_schema.tables", quote_presto_like_identifier(database))
};
format!(
let mut sql = format!(
"SELECT table_name, CASE table_type WHEN 'BASE TABLE' THEN 'TABLE' ELSE table_type END AS table_type \
FROM {source} \
WHERE table_schema = {} AND table_type IN ('BASE TABLE', 'VIEW') \
ORDER BY table_type, table_name",
WHERE table_schema = {} AND table_type IN ('BASE TABLE', 'VIEW')",
sql_string_literal(schema)
);
if let Some(filter) = filter.map(str::trim).filter(|value| !value.is_empty()) {
sql.push_str(" AND lower(table_name) LIKE ");
sql.push_str(&sql_string_literal(&format!("{}%", escape_presto_like_pattern(&filter.to_lowercase()))));
sql.push_str(" ESCAPE '\\'");
}
sql.push_str(" ORDER BY table_type, table_name");
if let Some(limit) = limit {
sql.push_str(&format!(" LIMIT {}", limit.max(1)));
}
sql
}
fn presto_like_information_schema_columns_sql(database: &str, schema: &str, table: &str) -> String {
let source = if database.trim().is_empty() {
"information_schema.columns".to_string()
} else {
format!("{}.information_schema.columns", quote_presto_like_identifier(database))
};
format!(
"SELECT column_name, data_type, is_nullable, column_default, comment \
FROM {source} \
WHERE table_schema = {} AND table_name = {} \
ORDER BY ordinal_position",
sql_string_literal(schema),
sql_string_literal(table)
)
}
@@ -1756,6 +1822,35 @@ fn presto_like_tables_from_query_result(result: &db::QueryResult) -> Vec<db::Tab
.collect()
}
fn presto_like_columns_from_query_result(result: &db::QueryResult) -> Vec<db::ColumnInfo> {
result
.rows
.iter()
.filter_map(|row| {
let name = query_result_cell_string(row, 0)?;
if name.trim().is_empty() {
return None;
}
let data_type = query_result_cell_string(row, 1).unwrap_or_default();
Some(db::ColumnInfo {
name,
// Presto/Trino do not expose precision/length columns in information_schema.columns.
data_type: data_type.clone(),
is_nullable: query_result_cell_string(row, 2)
.map(|value| value.eq_ignore_ascii_case("YES"))
.unwrap_or(true),
column_default: query_result_cell_string(row, 3),
is_primary_key: false,
extra: None,
comment: query_result_cell_string(row, 4),
numeric_precision: presto_like_numeric_precision(&data_type),
numeric_scale: presto_like_numeric_scale(&data_type),
character_maximum_length: presto_like_character_maximum_length(&data_type),
})
})
.collect()
}
fn query_result_cell_string(row: &[serde_json::Value], index: usize) -> Option<String> {
let value = row.get(index)?;
if value.is_null() {
@@ -1764,6 +1859,29 @@ fn query_result_cell_string(row: &[serde_json::Value], index: usize) -> Option<S
value.as_str().map(ToString::to_string).or_else(|| Some(value.to_string()))
}
fn presto_like_numeric_precision(data_type: &str) -> Option<i32> {
presto_like_type_argument(data_type, &["decimal", "numeric"], 0)
}
fn presto_like_numeric_scale(data_type: &str) -> Option<i32> {
presto_like_type_argument(data_type, &["decimal", "numeric"], 1)
}
fn presto_like_character_maximum_length(data_type: &str) -> Option<i32> {
presto_like_type_argument(data_type, &["char", "varchar"], 0)
}
fn presto_like_type_argument(data_type: &str, type_names: &[&str], index: usize) -> Option<i32> {
let value = data_type.trim();
let open = value.find('(')?;
let close = value[open + 1..].find(')')? + open + 1;
let name = value[..open].trim().to_ascii_lowercase();
if !type_names.iter().any(|type_name| *type_name == name) {
return None;
}
value[open + 1..close].split(',').nth(index)?.trim().parse::<i32>().ok()
}
fn normalize_information_schema_table_type(table_type: &str) -> String {
match table_type.trim().to_ascii_uppercase().replace(' ', "_").as_str() {
"BASE_TABLE" => "TABLE".to_string(),
@@ -1785,6 +1903,10 @@ fn sql_string_literal(value: &str) -> String {
format!("'{}'", value.replace('\'', "''"))
}
fn escape_presto_like_pattern(value: &str) -> String {
value.replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_")
}
#[cfg(test)]
mod tests {
use super::db;
@@ -1796,8 +1918,8 @@ mod tests {
oracle_object_statistics_from_query_result, oracle_object_statistics_rows_only_sql,
oracle_object_statistics_sql, oracle_object_statistics_user_segments_sql,
oracle_table_comment_from_query_result, oracle_table_comment_sql, oracle_table_comments_from_query_result,
oracle_table_comments_sql, presto_like_information_schema_tables_sql, presto_like_tables_from_query_result,
visible_schema_filter,
oracle_table_comments_sql, presto_like_columns_from_query_result, presto_like_information_schema_columns_sql,
presto_like_information_schema_tables_sql, presto_like_tables_from_query_result, visible_schema_filter,
};
#[cfg(feature = "duckdb-bundled")]
use super::{
@@ -2038,7 +2160,7 @@ mod tests {
#[test]
fn presto_like_information_schema_sql_uses_catalog_and_schema_without_system_jdbc() {
let sql = presto_like_information_schema_tables_sql("hive", "sales_analytics");
let sql = presto_like_information_schema_tables_sql("hive", "sales_analytics", None, None);
assert_eq!(
sql,
@@ -2049,12 +2171,40 @@ mod tests {
#[test]
fn presto_like_information_schema_sql_escapes_identifiers_and_literals() {
let sql = presto_like_information_schema_tables_sql("hi\"ve", "sales'analytics");
let sql = presto_like_information_schema_tables_sql("hi\"ve", "sales'analytics", None, None);
assert!(sql.contains("\"hi\"\"ve\".information_schema.tables"));
assert!(sql.contains("table_schema = 'sales''analytics'"));
}
#[test]
fn presto_like_information_schema_sql_pushes_table_filter_and_limit() {
let sql = presto_like_information_schema_tables_sql("hive", "sales_analytics", Some("Daily_%\\"), Some(20));
assert!(sql.contains("AND lower(table_name) LIKE 'daily\\_\\%\\\\%' ESCAPE '\\'"));
assert!(sql.ends_with("ORDER BY table_type, table_name LIMIT 20"));
}
#[test]
fn presto_like_information_schema_columns_sql_uses_catalog_information_schema() {
let sql = presto_like_information_schema_columns_sql("hive", "sales_analytics", "daily_revenue");
assert_eq!(
sql,
"SELECT column_name, data_type, is_nullable, column_default, comment FROM \"hive\".information_schema.columns WHERE table_schema = 'sales_analytics' AND table_name = 'daily_revenue' ORDER BY ordinal_position"
);
assert!(!sql.contains("system.jdbc.columns"));
}
#[test]
fn presto_like_information_schema_columns_sql_escapes_identifiers_and_literals() {
let sql = presto_like_information_schema_columns_sql("hi\"ve", "sales'analytics", "daily'revenue");
assert!(sql.contains("\"hi\"\"ve\".information_schema.columns"));
assert!(sql.contains("table_schema = 'sales''analytics'"));
assert!(sql.contains("table_name = 'daily''revenue'"));
}
#[test]
fn presto_like_tables_from_query_result_normalizes_base_table_type() {
let result = super::db::QueryResult {
@@ -2081,6 +2231,58 @@ mod tests {
assert_eq!(normalize_information_schema_table_type("MATERIALIZED VIEW"), "MATERIALIZED_VIEW");
}
#[test]
fn presto_like_columns_from_query_result_maps_column_metadata() {
let result = super::db::QueryResult {
columns: vec![
"column_name".to_string(),
"data_type".to_string(),
"is_nullable".to_string(),
"column_default".to_string(),
"comment".to_string(),
],
column_types: vec![],
column_sortables: vec![],
rows: vec![
vec![
serde_json::json!("amount"),
serde_json::json!("decimal(12,2)"),
serde_json::json!("NO"),
serde_json::Value::Null,
serde_json::json!("daily amount"),
],
vec![
serde_json::json!("code"),
serde_json::json!("varchar(64)"),
serde_json::json!("YES"),
serde_json::Value::Null,
serde_json::Value::Null,
],
],
affected_rows: 0,
execution_time_ms: 1,
truncated: false,
session_id: None,
has_more: false,
};
let columns = presto_like_columns_from_query_result(&result);
assert_eq!(columns[0].name, "amount");
assert_eq!(columns[0].data_type, "decimal(12,2)");
assert!(!columns[0].is_nullable);
assert_eq!(columns[0].comment.as_deref(), Some("daily amount"));
assert_eq!(columns[0].numeric_precision, Some(12));
assert_eq!(columns[0].numeric_scale, Some(2));
assert_eq!(columns[0].character_maximum_length, None);
assert!(!columns[0].is_primary_key);
assert_eq!(columns[1].name, "code");
assert!(columns[1].is_nullable);
assert_eq!(columns[1].numeric_precision, None);
assert_eq!(columns[1].numeric_scale, None);
assert_eq!(columns[1].character_maximum_length, Some(64));
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_list_databases_includes_attached_database() {
@@ -3167,6 +3369,9 @@ pub async fn get_columns_core(
let config = config.clone();
let session = session.clone();
drop(connections);
if uses_presto_like_information_schema_tables(&config.db_type) {
return external_driver_presto_like_columns(session, config.as_ref(), database, schema, table).await;
}
let columns = session
.invoke_with_timeout::<Vec<db::ColumnInfo>>(
"getColumns",