use std::collections::HashSet; use deadpool_postgres::Pool; use super::{ObjectInfo, TableInfo}; use crate::db; const CLOUD_BERRY_TABLE_DDL_SQL: &str = "SELECT pg_get_tabledef($1, $2, true)"; const CLOUD_BERRY_EXTERNAL_TABLES_SQL: &str = "SELECT c.relname \ FROM pg_catalog.pg_class c \ JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ WHERE n.nspname = $1 AND c.relkind = 'f' AND c.relname = ANY($2::text[])"; const CLOUD_BERRY_TABLE_MODIFIERS_SQL: &str = "SELECT COALESCE(am.amname, '')::text AS access_method, \ COALESCE(array_to_string(c.reloptions, E'\\n'), '')::text AS reloptions, \ COALESCE(dp.policytype::text, '')::text AS policy_type, \ COALESCE(string_agg(a.attname, E'\\n' \ ORDER BY array_position(dp.distkey::smallint[], a.attnum::smallint)), '')::text \ AS distribution_columns, \ bool_or(c.relkind = 'f') AS is_external, \ COALESCE(fs.srvname, '')::text AS external_server, \ ft.ftoptions AS external_options \ FROM pg_catalog.pg_class c \ JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam \ LEFT JOIN pg_catalog.gp_distribution_policy dp ON dp.localoid = c.oid \ LEFT JOIN pg_catalog.pg_attribute a ON a.attrelid = c.oid AND a.attnum = ANY(dp.distkey) \ LEFT JOIN pg_catalog.pg_foreign_table ft ON ft.ftrelid = c.oid \ LEFT JOIN pg_catalog.pg_foreign_server fs ON fs.oid = ft.ftserver \ WHERE n.nspname = $1 AND c.relname = $2 \ AND c.relkind IN ('r', 'p', 'f') \ GROUP BY c.oid, am.amname, c.reloptions, dp.policytype, dp.distkey, fs.srvname, ft.ftoptions \ LIMIT 1"; #[derive(Debug, Clone, PartialEq, Eq)] pub enum DistributionPolicy { Hash(Vec), Random, Replicated, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct ExternalTableDefinition { pub server: String, pub options: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct TableModifiers { pub access_method: Option, pub reloptions: Vec, pub distribution: Option, pub external: Option, } pub async fn list_tables_filtered( pool: &Pool, schema: &str, filter: Option<&str>, limit: Option, offset: Option, ) -> Result, String> { let mut tables = db::postgres::list_tables_filtered(pool, schema, filter, limit, offset).await?; annotate_external_tables(pool, schema, &mut tables).await; Ok(tables) } pub async fn list_objects(pool: &Pool, schema: &str) -> Result, String> { let mut objects = db::postgres::list_objects(pool, schema, true, true, false).await?; let names = objects.iter().map(|object| object.name.clone()).collect::>(); let external_names = external_table_names(pool, schema, &names).await.unwrap_or_else(|error| { log::debug!("[cloudberry][list_objects:external-table-fallback] error={error}"); HashSet::new() }); for object in &mut objects { if external_names.contains(&object.name) { object.object_type = "EXTERNAL TABLE".to_string(); } } Ok(objects) } pub async fn table_ddl(pool: &Pool, schema: &str, table: &str) -> Result { let client = db::postgres::checkout_postgres_client(pool, None, super::connection_timeout()).await?; let row = client .query_opt(CLOUD_BERRY_TABLE_DDL_SQL, &[&schema, &table]) .await .map_err(|error| error.to_string())? .ok_or_else(|| format!("Cloudberry table not found: {schema}.{table}"))?; let ddl = row.try_get::<_, Option>(0).map_err(|error| error.to_string())?.unwrap_or_default(); normalize_ddl(ddl) } pub async fn table_modifiers(pool: &Pool, schema: &str, table: &str) -> Result { let client = db::postgres::checkout_postgres_client(pool, None, super::connection_timeout()).await?; let row = client .query_opt(CLOUD_BERRY_TABLE_MODIFIERS_SQL, &[&schema, &table]) .await .map_err(|error| error.to_string())? .ok_or_else(|| format!("Cloudberry table not found: {schema}.{table}"))?; let access_method = non_empty(row.try_get::<_, String>(0).map_err(|error| error.to_string())?) .filter(|method| !method.eq_ignore_ascii_case("heap")); let reloptions = split_catalog_lines(row.try_get::<_, String>(1).map_err(|error| error.to_string())?); let policy_type = row.try_get::<_, String>(2).map_err(|error| error.to_string())?; let distribution_columns = split_catalog_lines(row.try_get::<_, String>(3).map_err(|error| error.to_string())?); let distribution = match policy_type.as_str() { "r" => Some(DistributionPolicy::Replicated), "p" if distribution_columns.is_empty() => Some(DistributionPolicy::Random), "p" => Some(DistributionPolicy::Hash(distribution_columns)), _ => None, }; let is_external = row.try_get::<_, bool>(4).map_err(|error| error.to_string())?; let external = if is_external { let server = row.try_get::<_, String>(5).map_err(|error| error.to_string())?; if server.trim().is_empty() { return Err(format!("Cloudberry external table has no foreign server: {schema}.{table}")); } Some(ExternalTableDefinition { server, options: row.try_get::<_, Option>>(6).map_err(|error| error.to_string())?.unwrap_or_default(), }) } else { None }; Ok(TableModifiers { access_method, reloptions, distribution, external }) } pub fn append_table_modifiers(ddl: &str, modifiers: &TableModifiers) -> Result { if let Some(external) = modifiers.external.as_ref() { return render_external_table_ddl(ddl, external); } let clauses = render_table_modifier_clauses(modifiers); if clauses.is_empty() { return Ok(ddl.to_string()); } let insertion = ddl .find(";\n") .or_else(|| ddl.find(';')) .ok_or_else(|| "Cloudberry fallback DDL has no CREATE TABLE terminator".to_string())?; let mut output = String::with_capacity(ddl.len() + clauses.len() + 2); output.push_str(&ddl[..insertion]); output.push('\n'); output.push_str(&clauses); output.push_str(&ddl[insertion..]); Ok(output) } fn render_external_table_ddl(ddl: &str, external: &ExternalTableDefinition) -> Result { // The fallback DDL may already read `CREATE FOREIGN TABLE` — a foreign // table's own relkind ('f') makes the generic renderer emit that header // directly — or plain `CREATE TABLE` from an older/other rendering path. // Accept either so this doesn't regress when the upstream header changes. let create_foreign_table = "CREATE FOREIGN TABLE "; let create_table = "CREATE TABLE "; let header_len = if ddl.starts_with(create_foreign_table) { create_foreign_table.len() } else if ddl.starts_with(create_table) { create_table.len() } else { return Err("Cloudberry external-table fallback expected CREATE TABLE DDL".to_string()); }; let insertion = ddl .find(";\n") .or_else(|| ddl.find(';')) .ok_or_else(|| "Cloudberry fallback DDL has no CREATE TABLE terminator".to_string())?; // When the fallback DDL already read `CREATE FOREIGN TABLE`, the generic // renderer also already appended its own ` SERVER ... [OPTIONS (...)]` // clause (from the same relkind='f' catalog data) right after the column // list's closing paren. Drop that so this doesn't emit the clause twice — // Cloudberry's own `external` definition above is authoritative here. // Locate the column list's own closing paren structurally (tracking // parenthesis depth and skipping quoted content) instead of searching for // the literal text `" SERVER \""`, which a column's DEFAULT/CHECK // expression could contain and match spuriously. let body_end = super::ddl_scan::find_top_level_paren_close(ddl, header_len).unwrap_or(insertion); let mut output = String::with_capacity(ddl.len() + external.options.len() * 24 + 48); output.push_str("CREATE FOREIGN TABLE "); output.push_str(&ddl[header_len..body_end]); output.push_str("\nSERVER "); output.push_str(&db::postgres::pg_quote_ident(&external.server)); if !external.options.is_empty() { // Cloudberry 2.x stores external tables as foreign tables. Reusing the // server options preserves URI, format and execution-location details. output.push_str("\nOPTIONS (\n "); output.push_str( &external .options .iter() .map(|option| render_foreign_table_option(option)) .collect::, _>>()? .join(",\n "), ); output.push_str("\n)"); } output.push_str(&ddl[insertion..]); Ok(output) } fn render_foreign_table_option(option: &str) -> Result { let (name, value) = option.split_once('=').ok_or_else(|| format!("Invalid Cloudberry foreign-table option: {option}"))?; Ok(format!("{} {}", db::postgres::pg_quote_ident(name), quote_sql_string(value))) } fn render_table_modifier_clauses(modifiers: &TableModifiers) -> String { let mut clauses = Vec::new(); if let Some(access_method) = modifiers.access_method.as_deref() { clauses.push(format!("USING {}", db::postgres::pg_quote_ident(access_method))); } if !modifiers.reloptions.is_empty() { clauses.push(format!("WITH (\n {}\n)", modifiers.reloptions.join(",\n "))); } if let Some(distribution) = modifiers.distribution.as_ref() { clauses.push(match distribution { DistributionPolicy::Hash(columns) => format!( "DISTRIBUTED BY ({})", columns.iter().map(|column| db::postgres::pg_quote_ident(column)).collect::>().join(", ") ), DistributionPolicy::Random => "DISTRIBUTED RANDOMLY".to_string(), DistributionPolicy::Replicated => "DISTRIBUTED REPLICATED".to_string(), }); } clauses.join("\n") } async fn annotate_external_tables(pool: &Pool, schema: &str, tables: &mut [TableInfo]) { let names = tables.iter().map(|table| table.name.clone()).collect::>(); let external_names = external_table_names(pool, schema, &names).await.unwrap_or_else(|error| { log::debug!("[cloudberry][list_tables:external-table-fallback] error={error}"); HashSet::new() }); for table in tables { if external_names.contains(&table.name) { table.table_type = "EXTERNAL TABLE".to_string(); } } } async fn external_table_names(pool: &Pool, schema: &str, names: &[String]) -> Result, String> { if names.is_empty() { return Ok(HashSet::new()); } let client = db::postgres::checkout_postgres_client(pool, None, super::connection_timeout()).await?; let rows = client.query(CLOUD_BERRY_EXTERNAL_TABLES_SQL, &[&schema, &names]).await.map_err(|error| error.to_string())?; Ok(rows.into_iter().filter_map(|row| row.try_get::<_, String>(0).ok()).collect()) } fn normalize_ddl(ddl: String) -> Result { let ddl = ddl.trim(); if ddl.is_empty() { return Err("Cloudberry returned an empty table DDL".to_string()); } if ddl.ends_with(';') { Ok(format!("{ddl}\n")) } else { Ok(format!("{ddl};\n")) } } fn non_empty(value: String) -> Option { let value = value.trim(); (!value.is_empty()).then(|| value.to_string()) } fn split_catalog_lines(value: String) -> Vec { value.lines().map(str::trim).filter(|value| !value.is_empty()).map(str::to_string).collect() } fn quote_sql_string(value: &str) -> String { format!("'{}'", value.replace('\'', "''")) } #[cfg(test)] mod tests { use super::*; fn modifiers(distribution: Option) -> TableModifiers { TableModifiers { access_method: None, reloptions: Vec::new(), distribution, external: None } } #[test] fn appends_hash_distribution_before_table_terminator() { let ddl = "CREATE TABLE \"public\".\"events\" (\n \"tenant_id\" integer\n);\n"; let rendered = append_table_modifiers( ddl, &modifiers(Some(DistributionPolicy::Hash(vec!["tenant_id".to_string(), "event id".to_string()]))), ) .unwrap(); assert_eq!( rendered, "CREATE TABLE \"public\".\"events\" (\n \"tenant_id\" integer\n)\nDISTRIBUTED BY (\"tenant_id\", \"event id\");\n" ); } #[test] fn appends_storage_and_replicated_distribution() { let ddl = "CREATE TABLE \"public\".\"dimensions\" (\n \"id\" integer\n);\n"; let rendered = append_table_modifiers( ddl, &TableModifiers { access_method: Some("ao_column".to_string()), reloptions: vec!["compresstype=zstd".to_string(), "compresslevel=3".to_string()], distribution: Some(DistributionPolicy::Replicated), external: None, }, ) .unwrap(); assert!(rendered.contains("USING \"ao_column\"")); assert!(rendered.contains("WITH (\n compresstype=zstd,\n compresslevel=3\n)")); assert!(rendered.contains("DISTRIBUTED REPLICATED;")); } #[test] fn renders_external_table_from_foreign_options() { let ddl = "CREATE TABLE \"public\".\"external_events\" (\n \"id\" integer\n);\n"; let rendered = append_table_modifiers( ddl, &TableModifiers { external: Some(ExternalTableDefinition { server: "gp_exttable_server".to_string(), options: vec![ "format=csv".to_string(), "location_uris=file://cdw/tmp/events.csv".to_string(), "null=".to_string(), ], }), ..modifiers(None) }, ) .unwrap(); assert!(rendered.starts_with("CREATE FOREIGN TABLE \"public\".\"external_events\"")); assert!(rendered.contains("SERVER \"gp_exttable_server\"")); assert!(rendered.contains("\"location_uris\" 'file://cdw/tmp/events.csv'")); assert!(rendered.contains("\"null\" ''")); } #[test] fn renders_external_table_from_already_foreign_ddl_without_duplicate_server_clause() { // The generic renderer now derives `is_foreign` straight from relkind // and may hand back a `CREATE FOREIGN TABLE ... SERVER ...` fallback // DDL directly, not just plain `CREATE TABLE`. let ddl = "CREATE FOREIGN TABLE \"public\".\"external_events\" (\n \"id\" integer\n) SERVER \"stale_server\";\n"; let rendered = append_table_modifiers( ddl, &TableModifiers { external: Some(ExternalTableDefinition { server: "gp_exttable_server".to_string(), options: vec!["format=csv".to_string()], }), ..modifiers(None) }, ) .unwrap(); assert!(rendered.starts_with("CREATE FOREIGN TABLE \"public\".\"external_events\"")); assert_eq!(rendered.matches("SERVER").count(), 1); assert!(rendered.contains("SERVER \"gp_exttable_server\"")); assert!(!rendered.contains("stale_server")); } #[test] fn renders_external_table_when_a_column_default_contains_literal_server_text() { // A naive text search for `" SERVER \""` to find the stale SERVER // clause's start would match this DEFAULT's embedded text instead of // the real trailing SERVER clause, truncating the column list. let ddl = "CREATE FOREIGN TABLE \"public\".\"external_events\" (\n \"id\" integer,\n \"note\" text DEFAULT 'contact SERVER \"admin\"'\n) SERVER \"stale_server\";\n"; let rendered = append_table_modifiers( ddl, &TableModifiers { external: Some(ExternalTableDefinition { server: "gp_exttable_server".to_string(), options: vec!["format=csv".to_string()], }), ..modifiers(None) }, ) .unwrap(); assert!(rendered.contains("\"note\" text DEFAULT 'contact SERVER \"admin\"'")); assert_eq!( rendered.matches("SERVER").count(), 2, "expected exactly the embedded literal plus the real clause: {rendered}" ); assert!(rendered.contains("SERVER \"gp_exttable_server\"")); assert!(!rendered.contains("stale_server")); } #[test] fn ddl_queries_avoid_privileged_external_catalog_view() { assert_eq!(CLOUD_BERRY_TABLE_DDL_SQL, "SELECT pg_get_tabledef($1, $2, true)"); assert!(CLOUD_BERRY_TABLE_MODIFIERS_SQL.contains("gp_distribution_policy")); assert!(CLOUD_BERRY_TABLE_MODIFIERS_SQL.contains("c.relkind = 'f'")); assert!(CLOUD_BERRY_EXTERNAL_TABLES_SQL.contains("c.relkind = 'f'")); assert!(!CLOUD_BERRY_TABLE_MODIFIERS_SQL.contains("pg_exttable")); assert!(!CLOUD_BERRY_EXTERNAL_TABLES_SQL.contains("pg_exttable")); } }