fix(presto): optimize metadata completion
This commit is contained in:
1 parent
f5f6bc0b43
commit
95e3cb6571
9 files changed
+528
-37
No files matched your search
+219
-14
@@ -1414,9 +1414,17 @@ async fn list_tables_once(
|
||||
let session = session.clone();
|
||||
drop(connections);
|
||||
if uses_presto_like_information_schema_tables(&config.db_type) {
|
||||
return external_driver_presto_like_tables(session, config.as_ref(), database, schema)
|
||||
.await
|
||||
.map(|tables| filter_table_infos(tables, filter, limit, offset, object_types));
|
||||
return external_driver_presto_like_tables(
|
||||
session,
|
||||
config.as_ref(),
|
||||
database,
|
||||
schema,
|
||||
filter,
|
||||
limit,
|
||||
offset,
|
||||
)
|
||||
.await
|
||||
.map(|tables| filter_table_infos(tables, filter, limit, offset, object_types));
|
||||
}
|
||||
return session
|
||||
.invoke_with_timeout::<Vec<db::TableInfo>>(
|
||||
@@ -1677,7 +1685,11 @@ async fn external_driver_presto_like_tables(
|
||||
config: &ConnectionConfig,
|
||||
database: &str,
|
||||
schema: &str,
|
||||
filter: Option<&str>,
|
||||
limit: Option<usize>,
|
||||
offset: Option<usize>,
|
||||
) -> Result<Vec<db::TableInfo>, String> {
|
||||
let query_limit = limit.map(|limit| limit.saturating_add(offset.unwrap_or(0)).max(1)).unwrap_or(100000);
|
||||
let result: db::QueryResult = session
|
||||
.invoke_with_timeout(
|
||||
"executeQuery",
|
||||
@@ -1685,8 +1697,8 @@ async fn external_driver_presto_like_tables(
|
||||
"connection": config,
|
||||
"database": database,
|
||||
"schema": schema,
|
||||
"sql": presto_like_information_schema_tables_sql(database, schema),
|
||||
"maxRows": 100000,
|
||||
"sql": presto_like_information_schema_tables_sql(database, schema, filter, Some(query_limit)),
|
||||
"maxRows": query_limit,
|
||||
"fetchSize": 1000,
|
||||
"timeoutSecs": 60
|
||||
}),
|
||||
@@ -1702,7 +1714,7 @@ async fn external_driver_presto_like_objects(
|
||||
database: &str,
|
||||
schema: &str,
|
||||
) -> Result<Vec<db::ObjectInfo>, String> {
|
||||
let tables = external_driver_presto_like_tables(session, config, database, schema).await?;
|
||||
let tables = external_driver_presto_like_tables(session, config, database, schema, None, None, None).await?;
|
||||
Ok(tables
|
||||
.into_iter()
|
||||
.map(|table| db::ObjectInfo {
|
||||
@@ -1719,18 +1731,72 @@ async fn external_driver_presto_like_objects(
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn presto_like_information_schema_tables_sql(database: &str, schema: &str) -> String {
|
||||
async fn external_driver_presto_like_columns(
|
||||
session: Arc<crate::plugins::PluginDriverSession>,
|
||||
config: &ConnectionConfig,
|
||||
database: &str,
|
||||
schema: &str,
|
||||
table: &str,
|
||||
) -> Result<Vec<db::ColumnInfo>, String> {
|
||||
let result: db::QueryResult = session
|
||||
.invoke(
|
||||
"executeQuery",
|
||||
serde_json::json!({
|
||||
"connection": config,
|
||||
"database": database,
|
||||
"schema": schema,
|
||||
"sql": presto_like_information_schema_columns_sql(database, schema, table),
|
||||
"maxRows": 10000,
|
||||
"fetchSize": 1000,
|
||||
"timeoutSecs": 60
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
Ok(presto_like_columns_from_query_result(&result))
|
||||
}
|
||||
|
||||
fn presto_like_information_schema_tables_sql(
|
||||
database: &str,
|
||||
schema: &str,
|
||||
filter: Option<&str>,
|
||||
limit: Option<usize>,
|
||||
) -> String {
|
||||
let source = if database.trim().is_empty() {
|
||||
"information_schema.tables".to_string()
|
||||
} else {
|
||||
format!("{}.information_schema.tables", quote_presto_like_identifier(database))
|
||||
};
|
||||
format!(
|
||||
let mut sql = format!(
|
||||
"SELECT table_name, CASE table_type WHEN 'BASE TABLE' THEN 'TABLE' ELSE table_type END AS table_type \
|
||||
FROM {source} \
|
||||
WHERE table_schema = {} AND table_type IN ('BASE TABLE', 'VIEW') \
|
||||
ORDER BY table_type, table_name",
|
||||
WHERE table_schema = {} AND table_type IN ('BASE TABLE', 'VIEW')",
|
||||
sql_string_literal(schema)
|
||||
);
|
||||
if let Some(filter) = filter.map(str::trim).filter(|value| !value.is_empty()) {
|
||||
sql.push_str(" AND lower(table_name) LIKE ");
|
||||
sql.push_str(&sql_string_literal(&format!("{}%", escape_presto_like_pattern(&filter.to_lowercase()))));
|
||||
sql.push_str(" ESCAPE '\\'");
|
||||
}
|
||||
sql.push_str(" ORDER BY table_type, table_name");
|
||||
if let Some(limit) = limit {
|
||||
sql.push_str(&format!(" LIMIT {}", limit.max(1)));
|
||||
}
|
||||
sql
|
||||
}
|
||||
|
||||
fn presto_like_information_schema_columns_sql(database: &str, schema: &str, table: &str) -> String {
|
||||
let source = if database.trim().is_empty() {
|
||||
"information_schema.columns".to_string()
|
||||
} else {
|
||||
format!("{}.information_schema.columns", quote_presto_like_identifier(database))
|
||||
};
|
||||
format!(
|
||||
"SELECT column_name, data_type, is_nullable, column_default, comment \
|
||||
FROM {source} \
|
||||
WHERE table_schema = {} AND table_name = {} \
|
||||
ORDER BY ordinal_position",
|
||||
sql_string_literal(schema),
|
||||
sql_string_literal(table)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1756,6 +1822,35 @@ fn presto_like_tables_from_query_result(result: &db::QueryResult) -> Vec<db::Tab
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn presto_like_columns_from_query_result(result: &db::QueryResult) -> Vec<db::ColumnInfo> {
|
||||
result
|
||||
.rows
|
||||
.iter()
|
||||
.filter_map(|row| {
|
||||
let name = query_result_cell_string(row, 0)?;
|
||||
if name.trim().is_empty() {
|
||||
return None;
|
||||
}
|
||||
let data_type = query_result_cell_string(row, 1).unwrap_or_default();
|
||||
Some(db::ColumnInfo {
|
||||
name,
|
||||
// Presto/Trino do not expose precision/length columns in information_schema.columns.
|
||||
data_type: data_type.clone(),
|
||||
is_nullable: query_result_cell_string(row, 2)
|
||||
.map(|value| value.eq_ignore_ascii_case("YES"))
|
||||
.unwrap_or(true),
|
||||
column_default: query_result_cell_string(row, 3),
|
||||
is_primary_key: false,
|
||||
extra: None,
|
||||
comment: query_result_cell_string(row, 4),
|
||||
numeric_precision: presto_like_numeric_precision(&data_type),
|
||||
numeric_scale: presto_like_numeric_scale(&data_type),
|
||||
character_maximum_length: presto_like_character_maximum_length(&data_type),
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn query_result_cell_string(row: &[serde_json::Value], index: usize) -> Option<String> {
|
||||
let value = row.get(index)?;
|
||||
if value.is_null() {
|
||||
@@ -1764,6 +1859,29 @@ fn query_result_cell_string(row: &[serde_json::Value], index: usize) -> Option<S
|
||||
value.as_str().map(ToString::to_string).or_else(|| Some(value.to_string()))
|
||||
}
|
||||
|
||||
fn presto_like_numeric_precision(data_type: &str) -> Option<i32> {
|
||||
presto_like_type_argument(data_type, &["decimal", "numeric"], 0)
|
||||
}
|
||||
|
||||
fn presto_like_numeric_scale(data_type: &str) -> Option<i32> {
|
||||
presto_like_type_argument(data_type, &["decimal", "numeric"], 1)
|
||||
}
|
||||
|
||||
fn presto_like_character_maximum_length(data_type: &str) -> Option<i32> {
|
||||
presto_like_type_argument(data_type, &["char", "varchar"], 0)
|
||||
}
|
||||
|
||||
fn presto_like_type_argument(data_type: &str, type_names: &[&str], index: usize) -> Option<i32> {
|
||||
let value = data_type.trim();
|
||||
let open = value.find('(')?;
|
||||
let close = value[open + 1..].find(')')? + open + 1;
|
||||
let name = value[..open].trim().to_ascii_lowercase();
|
||||
if !type_names.iter().any(|type_name| *type_name == name) {
|
||||
return None;
|
||||
}
|
||||
value[open + 1..close].split(',').nth(index)?.trim().parse::<i32>().ok()
|
||||
}
|
||||
|
||||
fn normalize_information_schema_table_type(table_type: &str) -> String {
|
||||
match table_type.trim().to_ascii_uppercase().replace(' ', "_").as_str() {
|
||||
"BASE_TABLE" => "TABLE".to_string(),
|
||||
@@ -1785,6 +1903,10 @@ fn sql_string_literal(value: &str) -> String {
|
||||
format!("'{}'", value.replace('\'', "''"))
|
||||
}
|
||||
|
||||
fn escape_presto_like_pattern(value: &str) -> String {
|
||||
value.replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::db;
|
||||
@@ -1796,8 +1918,8 @@ mod tests {
|
||||
oracle_object_statistics_from_query_result, oracle_object_statistics_rows_only_sql,
|
||||
oracle_object_statistics_sql, oracle_object_statistics_user_segments_sql,
|
||||
oracle_table_comment_from_query_result, oracle_table_comment_sql, oracle_table_comments_from_query_result,
|
||||
oracle_table_comments_sql, presto_like_information_schema_tables_sql, presto_like_tables_from_query_result,
|
||||
visible_schema_filter,
|
||||
oracle_table_comments_sql, presto_like_columns_from_query_result, presto_like_information_schema_columns_sql,
|
||||
presto_like_information_schema_tables_sql, presto_like_tables_from_query_result, visible_schema_filter,
|
||||
};
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
use super::{
|
||||
@@ -2038,7 +2160,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn presto_like_information_schema_sql_uses_catalog_and_schema_without_system_jdbc() {
|
||||
let sql = presto_like_information_schema_tables_sql("hive", "sales_analytics");
|
||||
let sql = presto_like_information_schema_tables_sql("hive", "sales_analytics", None, None);
|
||||
|
||||
assert_eq!(
|
||||
sql,
|
||||
@@ -2049,12 +2171,40 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn presto_like_information_schema_sql_escapes_identifiers_and_literals() {
|
||||
let sql = presto_like_information_schema_tables_sql("hi\"ve", "sales'analytics");
|
||||
let sql = presto_like_information_schema_tables_sql("hi\"ve", "sales'analytics", None, None);
|
||||
|
||||
assert!(sql.contains("\"hi\"\"ve\".information_schema.tables"));
|
||||
assert!(sql.contains("table_schema = 'sales''analytics'"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn presto_like_information_schema_sql_pushes_table_filter_and_limit() {
|
||||
let sql = presto_like_information_schema_tables_sql("hive", "sales_analytics", Some("Daily_%\\"), Some(20));
|
||||
|
||||
assert!(sql.contains("AND lower(table_name) LIKE 'daily\\_\\%\\\\%' ESCAPE '\\'"));
|
||||
assert!(sql.ends_with("ORDER BY table_type, table_name LIMIT 20"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn presto_like_information_schema_columns_sql_uses_catalog_information_schema() {
|
||||
let sql = presto_like_information_schema_columns_sql("hive", "sales_analytics", "daily_revenue");
|
||||
|
||||
assert_eq!(
|
||||
sql,
|
||||
"SELECT column_name, data_type, is_nullable, column_default, comment FROM \"hive\".information_schema.columns WHERE table_schema = 'sales_analytics' AND table_name = 'daily_revenue' ORDER BY ordinal_position"
|
||||
);
|
||||
assert!(!sql.contains("system.jdbc.columns"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn presto_like_information_schema_columns_sql_escapes_identifiers_and_literals() {
|
||||
let sql = presto_like_information_schema_columns_sql("hi\"ve", "sales'analytics", "daily'revenue");
|
||||
|
||||
assert!(sql.contains("\"hi\"\"ve\".information_schema.columns"));
|
||||
assert!(sql.contains("table_schema = 'sales''analytics'"));
|
||||
assert!(sql.contains("table_name = 'daily''revenue'"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn presto_like_tables_from_query_result_normalizes_base_table_type() {
|
||||
let result = super::db::QueryResult {
|
||||
@@ -2081,6 +2231,58 @@ mod tests {
|
||||
assert_eq!(normalize_information_schema_table_type("MATERIALIZED VIEW"), "MATERIALIZED_VIEW");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn presto_like_columns_from_query_result_maps_column_metadata() {
|
||||
let result = super::db::QueryResult {
|
||||
columns: vec![
|
||||
"column_name".to_string(),
|
||||
"data_type".to_string(),
|
||||
"is_nullable".to_string(),
|
||||
"column_default".to_string(),
|
||||
"comment".to_string(),
|
||||
],
|
||||
column_types: vec![],
|
||||
column_sortables: vec![],
|
||||
rows: vec![
|
||||
vec![
|
||||
serde_json::json!("amount"),
|
||||
serde_json::json!("decimal(12,2)"),
|
||||
serde_json::json!("NO"),
|
||||
serde_json::Value::Null,
|
||||
serde_json::json!("daily amount"),
|
||||
],
|
||||
vec![
|
||||
serde_json::json!("code"),
|
||||
serde_json::json!("varchar(64)"),
|
||||
serde_json::json!("YES"),
|
||||
serde_json::Value::Null,
|
||||
serde_json::Value::Null,
|
||||
],
|
||||
],
|
||||
affected_rows: 0,
|
||||
execution_time_ms: 1,
|
||||
truncated: false,
|
||||
session_id: None,
|
||||
has_more: false,
|
||||
};
|
||||
|
||||
let columns = presto_like_columns_from_query_result(&result);
|
||||
|
||||
assert_eq!(columns[0].name, "amount");
|
||||
assert_eq!(columns[0].data_type, "decimal(12,2)");
|
||||
assert!(!columns[0].is_nullable);
|
||||
assert_eq!(columns[0].comment.as_deref(), Some("daily amount"));
|
||||
assert_eq!(columns[0].numeric_precision, Some(12));
|
||||
assert_eq!(columns[0].numeric_scale, Some(2));
|
||||
assert_eq!(columns[0].character_maximum_length, None);
|
||||
assert!(!columns[0].is_primary_key);
|
||||
assert_eq!(columns[1].name, "code");
|
||||
assert!(columns[1].is_nullable);
|
||||
assert_eq!(columns[1].numeric_precision, None);
|
||||
assert_eq!(columns[1].numeric_scale, None);
|
||||
assert_eq!(columns[1].character_maximum_length, Some(64));
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[test]
|
||||
fn duckdb_list_databases_includes_attached_database() {
|
||||
@@ -3167,6 +3369,9 @@ pub async fn get_columns_core(
|
||||
let config = config.clone();
|
||||
let session = session.clone();
|
||||
drop(connections);
|
||||
if uses_presto_like_information_schema_tables(&config.db_type) {
|
||||
return external_driver_presto_like_columns(session, config.as_ref(), database, schema, table).await;
|
||||
}
|
||||
let columns = session
|
||||
.invoke_with_timeout::<Vec<db::ColumnInfo>>(
|
||||
"getColumns",
|
||||
|
||||
Reference in new issue
Block a user