diff --git a/crates/dbx-core/src/db/cloudberry.rs b/crates/dbx-core/src/db/cloudberry.rs index e043030c2e..99c81021fc 100644 --- a/crates/dbx-core/src/db/cloudberry.rs +++ b/crates/dbx-core/src/db/cloudberry.rs @@ -170,10 +170,14 @@ fn render_external_table_ddl(ddl: &str, external: &ExternalTableDefinition) -> R .ok_or_else(|| "Cloudberry fallback DDL has no CREATE TABLE terminator".to_string())?; // When the fallback DDL already read `CREATE FOREIGN TABLE`, the generic // renderer also already appended its own ` SERVER ... [OPTIONS (...)]` - // clause (from the same relkind='f' catalog data). Drop that so this - // doesn't emit the clause twice — Cloudberry's own `external` definition - // above is authoritative here. - let body_end = ddl[header_len..insertion].find(" SERVER \"").map(|pos| header_len + pos).unwrap_or(insertion); + // clause (from the same relkind='f' catalog data) right after the column + // list's closing paren. Drop that so this doesn't emit the clause twice — + // Cloudberry's own `external` definition above is authoritative here. + // Locate the column list's own closing paren structurally (tracking + // parenthesis depth and skipping quoted content) instead of searching for + // the literal text `" SERVER \""`, which a column's DEFAULT/CHECK + // expression could contain and match spuriously. + let body_end = super::ddl_scan::find_top_level_paren_close(ddl, header_len).unwrap_or(insertion); let mut output = String::with_capacity(ddl.len() + external.options.len() * 24 + 48); output.push_str("CREATE FOREIGN TABLE "); output.push_str(&ddl[header_len..body_end]); @@ -364,6 +368,34 @@ mod tests { assert!(!rendered.contains("stale_server")); } + #[test] + fn renders_external_table_when_a_column_default_contains_literal_server_text() { + // A naive text search for `" SERVER \""` to find the stale SERVER + // clause's start would match this DEFAULT's embedded text instead of + // the real trailing SERVER clause, truncating the column list. + let ddl = "CREATE FOREIGN TABLE \"public\".\"external_events\" (\n \"id\" integer,\n \"note\" text DEFAULT 'contact SERVER \"admin\"'\n) SERVER \"stale_server\";\n"; + let rendered = append_table_modifiers( + ddl, + &TableModifiers { + external: Some(ExternalTableDefinition { + server: "gp_exttable_server".to_string(), + options: vec!["format=csv".to_string()], + }), + ..modifiers(None) + }, + ) + .unwrap(); + + assert!(rendered.contains("\"note\" text DEFAULT 'contact SERVER \"admin\"'")); + assert_eq!( + rendered.matches("SERVER").count(), + 2, + "expected exactly the embedded literal plus the real clause: {rendered}" + ); + assert!(rendered.contains("SERVER \"gp_exttable_server\"")); + assert!(!rendered.contains("stale_server")); + } + #[test] fn ddl_queries_avoid_privileged_external_catalog_view() { assert_eq!(CLOUD_BERRY_TABLE_DDL_SQL, "SELECT pg_get_tabledef($1, $2, true)"); diff --git a/crates/dbx-core/src/db/ddl_scan.rs b/crates/dbx-core/src/db/ddl_scan.rs new file mode 100644 index 0000000000..0d3f8634a1 --- /dev/null +++ b/crates/dbx-core/src/db/ddl_scan.rs @@ -0,0 +1,223 @@ +//! Structural, quote-and-parenthesis-aware scanning helpers for +//! post-processing already-rendered `CREATE TABLE` DDL text. +//! +//! A naive `str::find` for a literal marker (e.g. `" SERVER \""` or a bare +//! `;`) can be fooled by that same text appearing inside a quoted string +//! literal, a quoted identifier, or a nested parenthesized sub-expression — +//! a column's `DEFAULT`/`CHECK` expression can legally contain any of those. +//! These helpers track quote and parenthesis depth so callers only ever +//! match at the top level. + +use std::ops::Range; + +fn char_at(sql: &str, pos: usize) -> Option { + sql.get(pos..)?.chars().next() +} + +/// Skips a `'...'` string literal (`''`-escaped, with backslash-escape +/// tolerance) or a `"..."` quoted identifier (`""`-escaped) starting at +/// `pos`, which must point at the opening `quote` character. Returns the +/// index just past the closing quote (or `sql.len()` if unterminated). +fn skip_quoted(sql: &str, pos: usize, quote: char) -> usize { + let mut i = pos + quote.len_utf8(); + while i < sql.len() { + let Some(ch) = char_at(sql, i) else { break }; + let next = char_at(sql, i + ch.len_utf8()); + if ch == quote { + if next == Some(quote) { + i += ch.len_utf8() + quote.len_utf8(); + continue; + } + return i + ch.len_utf8(); + } + if quote == '\'' && ch == '\\' { + i += ch.len_utf8(); + if let Some(escaped) = char_at(sql, i) { + i += escaped.len_utf8(); + } + continue; + } + i += ch.len_utf8(); + } + sql.len() +} + +/// Skips a PostgreSQL `$tag$...$tag$` dollar-quoted body starting at `pos` +/// (pointing at the first `$`). Returns `None` if `pos` isn't actually the +/// start of a valid dollar-quote tag (e.g. a stray `$` in an expression), in +/// which case the caller should treat `$` as an ordinary character. +fn skip_dollar_quoted(sql: &str, pos: usize) -> Option { + let tag_end_offset = sql.get(pos + 1..)?.find('$')?; + let tag_end = pos + 1 + tag_end_offset; + let tag = &sql[pos + 1..tag_end]; + let valid_tag = tag.is_empty() + || (tag.chars().next().is_some_and(|ch| ch.is_ascii_alphabetic() || ch == '_') + && tag.chars().all(|ch| ch.is_ascii_alphanumeric() || ch == '_')); + if !valid_tag { + return None; + } + let delimiter = &sql[pos..=tag_end]; + let content_start = tag_end + 1; + sql.get(content_start..)?.find(delimiter).map(|offset| content_start + offset + delimiter.len()) +} + +/// Finds the byte index just past the first top-level (parenthesis-depth-0) +/// closing paren at or after `from` — the end of the outermost parenthesized +/// group that starts somewhere at or after `from` — treating quoted or +/// dollar-quoted content as opaque so parens or keywords inside a +/// `DEFAULT`/`CHECK` expression can't confuse the count. Returns `None` if no +/// top-level `(` is ever opened, or it's never closed. +pub(crate) fn find_top_level_paren_close(sql: &str, from: usize) -> Option { + let mut i = from; + let mut depth: usize = 0; + let mut opened = false; + while i < sql.len() { + let ch = char_at(sql, i)?; + match ch { + '\'' | '"' => { + i = skip_quoted(sql, i, ch); + continue; + } + '$' => { + if let Some(end) = skip_dollar_quoted(sql, i) { + i = end; + continue; + } + } + '(' => { + depth += 1; + opened = true; + } + ')' => { + depth = depth.saturating_sub(1); + i += ch.len_utf8(); + if opened && depth == 0 { + return Some(i); + } + continue; + } + _ => {} + } + i += ch.len_utf8(); + } + None +} + +/// Finds the byte index of the next top-level (depth-0, outside any quoted +/// or dollar-quoted content) `;` at or after `from`. +fn find_top_level_semicolon(sql: &str, from: usize) -> Option { + let mut i = from; + let mut depth: usize = 0; + while i < sql.len() { + let ch = char_at(sql, i)?; + match ch { + '\'' | '"' => { + i = skip_quoted(sql, i, ch); + continue; + } + '$' => { + if let Some(end) = skip_dollar_quoted(sql, i) { + i = end; + continue; + } + } + '(' => depth += 1, + ')' => depth = depth.saturating_sub(1), + ';' if depth == 0 => return Some(i), + _ => {} + } + i += ch.len_utf8(); + } + None +} + +/// Splits `sql` into consecutive top-level statement ranges, each ending +/// just past its own top-level `;`. The last statement may lack a trailing +/// `;`, in which case its range runs to the end of the string. +pub(crate) fn top_level_statement_ranges(sql: &str) -> Vec> { + let mut ranges = Vec::new(); + let mut start = 0; + while start < sql.len() { + match find_top_level_semicolon(sql, start) { + Some(semi) => { + ranges.push(start..semi + 1); + start = semi + 1; + } + None => { + ranges.push(start..sql.len()); + break; + } + } + } + ranges +} + +/// If `stmt` (ignoring leading whitespace) begins with `CREATE TABLE +/// "schema"."table"` or `CREATE FOREIGN TABLE "schema"."table"`, returns the +/// unescaped `(schema, table)` pair. +pub(crate) fn parse_create_table_relation(stmt: &str) -> Option<(String, String)> { + let trimmed = stmt.trim_start(); + let rest = + trimmed.strip_prefix("CREATE FOREIGN TABLE ").or_else(|| trimmed.strip_prefix("CREATE TABLE "))?.trim_start(); + let (schema, rest) = unquote_ident(rest)?; + let (table, _) = unquote_ident(rest.strip_prefix('.')?)?; + Some((schema, table)) +} + +fn unquote_ident(s: &str) -> Option<(String, &str)> { + if !s.starts_with('"') { + return None; + } + let end = skip_quoted(s, 0, '"'); + if end < 2 { + return None; + } + let raw = &s[1..end - 1]; + Some((raw.replace("\"\"", "\""), &s[end..])) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn paren_close_skips_nested_parens_and_quotes() { + let ddl = + "\"public\".\"t\" (\n \"a\" text DEFAULT 'x(y)'::text,\n \"b\" text CHECK ((b <> 'z'))\n) SERVER \"s\""; + let close = find_top_level_paren_close(ddl, 0).unwrap(); + assert_eq!( + &ddl[..close], + "\"public\".\"t\" (\n \"a\" text DEFAULT 'x(y)'::text,\n \"b\" text CHECK ((b <> 'z'))\n)" + ); + } + + #[test] + fn paren_close_ignores_literal_server_text_inside_quotes() { + let ddl = "\"public\".\"t\" (\n \"note\" text DEFAULT 'contact SERVER \"admin\"'\n) SERVER \"real_server\""; + let close = find_top_level_paren_close(ddl, 0).unwrap(); + assert!(ddl[close..].starts_with(" SERVER \"real_server\"")); + } + + #[test] + fn top_level_statement_ranges_split_on_depth_zero_semicolons() { + let ddl = "CREATE TABLE \"s\".\"t\" (\n \"a\" text DEFAULT 'x;y'\n);\nALTER TABLE \"s\".\"t\" ADD CONSTRAINT c CHECK (a <> '');"; + let ranges = top_level_statement_ranges(ddl); + assert_eq!(ranges.len(), 2); + assert!(ddl[ranges[0].clone()].starts_with("CREATE TABLE")); + assert!(ddl[ranges[0].clone()].ends_with(";")); + assert!(ddl[ranges[1].clone()].trim_start().starts_with("ALTER TABLE")); + } + + #[test] + fn parse_create_table_relation_unescapes_doubled_quotes() { + let (schema, table) = + parse_create_table_relation("CREATE TABLE \"pub\"\"lic\".\"t\" (\n \"a\" int\n);").unwrap(); + assert_eq!(schema, "pub\"lic"); + assert_eq!(table, "t"); + } + + #[test] + fn parse_create_table_relation_rejects_non_create_table() { + assert!(parse_create_table_relation("ALTER TABLE \"s\".\"t\" ADD CONSTRAINT c CHECK (true);").is_none()); + } +} diff --git a/crates/dbx-core/src/db/mod.rs b/crates/dbx-core/src/db/mod.rs index 5251a74b65..755c9eb90e 100644 --- a/crates/dbx-core/src/db/mod.rs +++ b/crates/dbx-core/src/db/mod.rs @@ -3,6 +3,7 @@ pub mod clickhouse_driver; pub mod cloudberry; pub mod cloudflare_d1; pub use cloudflare_d1 as cloudflare_d1_driver; +pub(crate) mod ddl_scan; pub mod document_result; pub mod dolt; pub mod doris; diff --git a/crates/dbx-core/src/db/opentenbase.rs b/crates/dbx-core/src/db/opentenbase.rs index 573cfeb87e..87c40fd50e 100644 --- a/crates/dbx-core/src/db/opentenbase.rs +++ b/crates/dbx-core/src/db/opentenbase.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use deadpool_postgres::Pool; use crate::db; @@ -20,6 +22,24 @@ const TABLE_DISTRIBUTION_SQL: &str = "SELECT c.pclocatortype::text, \ WHERE namespace.nspname = $1 AND relation.relname = $2 \ LIMIT 1"; +/// Batched sibling of `TABLE_DISTRIBUTION_SQL`, for appending each +/// partition's own `DISTRIBUTE BY` clause when rendering a whole partition +/// tree instead of just the root relation. +const TABLE_DISTRIBUTION_FOR_RELATIONS_SQL: &str = + "SELECT namespace.nspname, relation.relname, c.pclocatortype::text, \ + COALESCE(( \ + SELECT string_agg(a.attname, E'\\n' ORDER BY distributed_column.ordinality) \ + FROM unnest(c.discolnums::smallint[]) WITH ORDINALITY \ + AS distributed_column(attnum, ordinality) \ + JOIN pg_catalog.pg_attribute a \ + ON a.attrelid = c.pcrelid AND a.attnum = distributed_column.attnum \ + WHERE NOT a.attisdropped \ + ), '')::text \ + FROM unnest($1::text[], $2::text[]) AS rel(rel_schema, rel_table) \ + JOIN pg_catalog.pg_namespace namespace ON namespace.nspname = rel.rel_schema \ + JOIN pg_catalog.pg_class relation ON relation.relnamespace = namespace.oid AND relation.relname = rel.rel_table \ + JOIN pg_catalog.pgxc_class c ON c.pcrelid = relation.oid"; + #[derive(Debug, Clone, PartialEq, Eq)] pub enum DistributionPolicy { Hash(Vec), @@ -52,6 +72,43 @@ pub async fn table_distribution(pool: &Pool, schema: &str, table: &str) -> Resul parse_distribution_policy(&locator_type, columns) } +/// Batched sibling of `table_distribution`, for a whole partition tree. +/// Relations without a distribution policy (or that don't exist) are simply +/// absent from the result map rather than erroring the whole batch. +pub async fn table_distribution_for_relations( + pool: &Pool, + relations: &[(String, String)], +) -> Result, String> { + if relations.is_empty() { + return Ok(HashMap::new()); + } + let client = db::postgres::checkout_postgres_client(pool, None, super::connection_timeout()).await?; + let schemas: Vec<&str> = relations.iter().map(|(schema, _)| schema.as_str()).collect(); + let tables: Vec<&str> = relations.iter().map(|(_, table)| table.as_str()).collect(); + let rows = client + .query(TABLE_DISTRIBUTION_FOR_RELATIONS_SQL, &[&schemas, &tables]) + .await + .map_err(|error| error.to_string())?; + let mut result = HashMap::new(); + for row in &rows { + let schema = row.try_get::<_, String>(0).map_err(|error| error.to_string())?; + let table = row.try_get::<_, String>(1).map_err(|error| error.to_string())?; + let locator_type = row.try_get::<_, String>(2).map_err(|error| error.to_string())?; + let columns = row + .try_get::<_, String>(3) + .map_err(|error| error.to_string())? + .lines() + .map(str::trim) + .filter(|column| !column.is_empty()) + .map(str::to_string) + .collect::>(); + if let Some(distribution) = parse_distribution_policy(&locator_type, columns)? { + result.insert((schema, table), distribution); + } + } + Ok(result) +} + pub fn append_distribution_clause(ddl: &str, distribution: &DistributionPolicy) -> Result { let clause = render_distribution_clause(distribution)?; let insertion = ddl @@ -66,6 +123,41 @@ pub fn append_distribution_clause(ddl: &str, distribution: &DistributionPolicy) Ok(output) } +/// Tree-aware sibling of `append_distribution_clause`: `ddl` may contain +/// several relations' worth of statements concatenated together (root plus +/// every partition), so each `CREATE [FOREIGN] TABLE "schema"."table"` +/// statement gets its own `DISTRIBUTE BY` clause inserted before its own +/// terminator, looked up by that statement's own `(schema, table)` — instead +/// of inserting a single clause at the first `;` in the whole string, which +/// would only ever land on the root relation's statement. +pub fn append_distribution_clauses( + ddl: &str, + distributions: &HashMap<(String, String), DistributionPolicy>, +) -> Result { + if distributions.is_empty() { + return Ok(ddl.to_string()); + } + let mut output = String::with_capacity(ddl.len() + distributions.len() * 32); + let mut cursor = 0usize; + for range in db::ddl_scan::top_level_statement_ranges(ddl) { + let statement = &ddl[range.clone()]; + let Some(relation) = db::ddl_scan::parse_create_table_relation(statement) else { + continue; + }; + let Some(distribution) = distributions.get(&relation) else { + continue; + }; + let clause = render_distribution_clause(distribution)?; + let insertion = if statement.ends_with(';') { range.end - 1 } else { range.end }; + output.push_str(&ddl[cursor..insertion]); + output.push('\n'); + output.push_str(&clause); + cursor = insertion; + } + output.push_str(&ddl[cursor..]); + Ok(output) +} + fn parse_distribution_policy(locator_type: &str, columns: Vec) -> Result, String> { let column_distribution = |kind| { if columns.is_empty() { @@ -148,4 +240,32 @@ mod tests { assert!(TABLE_DISTRIBUTION_SQL.contains("WITH ORDINALITY")); assert!(TABLE_DISTRIBUTION_SQL.contains("NOT a.attisdropped")); } + + #[test] + fn batched_distribution_query_matches_opentenbase_catalog_contract() { + assert!(TABLE_DISTRIBUTION_FOR_RELATIONS_SQL.contains("unnest($1::text[], $2::text[])")); + assert!(TABLE_DISTRIBUTION_FOR_RELATIONS_SQL.contains("pg_catalog.pgxc_class")); + assert!(TABLE_DISTRIBUTION_FOR_RELATIONS_SQL.contains("WITH ORDINALITY")); + } + + #[test] + fn appends_each_partition_its_own_distribution_clause_in_a_multi_relation_tree() { + let ddl = "CREATE TABLE \"public\".\"events\" (\n \"id\" integer\n) PARTITION BY RANGE (id);\n\nCREATE TABLE \"public\".\"events_2026\" PARTITION OF \"public\".\"events\" FOR VALUES FROM (1) TO (100);\n\nCREATE TABLE \"public\".\"events_2027\" PARTITION OF \"public\".\"events\" FOR VALUES FROM (100) TO (200);\n"; + let mut distributions = HashMap::new(); + distributions + .insert(("public".to_string(), "events".to_string()), DistributionPolicy::Hash(vec!["id".to_string()])); + distributions.insert(("public".to_string(), "events_2027".to_string()), DistributionPolicy::Replication); + // "events_2026" deliberately has no entry — it must be left untouched. + + let rendered = append_distribution_clauses(ddl, &distributions).unwrap(); + + assert!(rendered.contains("PARTITION BY RANGE (id)\nDISTRIBUTE BY HASH (\"id\");")); + assert!(rendered.contains( + "CREATE TABLE \"public\".\"events_2027\" PARTITION OF \"public\".\"events\" FOR VALUES FROM (100) TO (200)\nDISTRIBUTE BY REPLICATION;" + )); + assert!(rendered.contains( + "CREATE TABLE \"public\".\"events_2026\" PARTITION OF \"public\".\"events\" FOR VALUES FROM (1) TO (100);" + )); + assert_eq!(rendered.matches("DISTRIBUTE BY").count(), 2); + } } diff --git a/crates/dbx-core/src/db/postgres.rs b/crates/dbx-core/src/db/postgres.rs index 87952f5826..aba199f390 100644 --- a/crates/dbx-core/src/db/postgres.rs +++ b/crates/dbx-core/src/db/postgres.rs @@ -1018,6 +1018,37 @@ fn pg_error_to_string(err: tokio_postgres::Error) -> String { err.as_db_error().map(ToString::to_string).unwrap_or_else(|| err.to_string()) } +/// Tries each SQL tier in `tiers` in order (most-capable first), via `run`, +/// returning the first tier that succeeds. Every driver-compat query in this +/// module (a "does this server have the newer catalog column" primary/compat +/// split, occasionally with a further information_schema fallback) used to +/// hand-roll this same try/log/combine-errors shape once per query; this is +/// the shared version. +/// +/// If every tier fails, all of their errors are logged together at debug +/// level (so a fallback firing in production is diagnosable) and the last +/// tier's error is returned to the caller, since it's usually the most +/// specific one for whatever the connected server actually is. +async fn query_with_compat_fallback( + log_context: &str, + tiers: &[&'static str], + mut run: F, +) -> Result +where + F: FnMut(&'static str) -> Fut, + Fut: std::future::Future>, +{ + let mut errors: Vec = Vec::new(); + for sql in tiers { + match run(sql).await { + Ok(value) => return Ok(value), + Err(error) => errors.push(pg_error_to_string(error)), + } + } + log::debug!("[postgres][{log_context}:compat-failed] {}", errors.join("; ")); + Err(errors.into_iter().next_back().unwrap_or_else(|| format!("[postgres][{log_context}] no SQL tiers configured"))) +} + fn pg_db_error_to_string(err: &tokio_postgres::error::DbError) -> String { format!("{err} (SQLSTATE {})", err.code().code()) } @@ -3243,23 +3274,11 @@ pub async fn get_columns_for_relations( ) -> Result>, String> { let oids: Vec = relations.iter().map(|(oid, _, _)| *oid).collect(); let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?; - let [primary_sql, compat_sql] = postgres_columns_for_relations_query_tiers(); - match get_columns_for_relations_with_sql(&client, primary_sql, &oids).await { - Ok(columns) => Ok(columns), - Err(primary_error) => match get_columns_for_relations_with_sql(&client, compat_sql, &oids).await { - Ok(columns) => Ok(columns), - Err(fallback_error) => { - let primary_message = pg_error_to_string(primary_error); - let fallback_message = pg_error_to_string(fallback_error); - log::debug!( - "[postgres][get_columns_for_relations:compat-failed] primary_error={} fallback_error={}", - primary_message, - fallback_message - ); - Err(fallback_message) - } - }, - } + let tiers = postgres_columns_for_relations_query_tiers(); + query_with_compat_fallback("get_columns_for_relations", &tiers, |sql| { + get_columns_for_relations_with_sql(&client, sql, &oids) + }) + .await } async fn get_columns_for_relations_with_sql( @@ -3280,6 +3299,15 @@ fn postgres_columns_for_relations_query_tiers() -> [&'static str; 2] { [postgres_columns_for_relations_sql(), postgres_columns_for_relations_compat_sql()] } +// Sibling of `POSTGRES_COLUMNS_SQL`/`POSTGRES_COLUMNS_COMPAT_SQL` below (~line +// 4880): same column list and detection logic (identity/serial inference, +// numeric precision/scale, enum values), batched by oid instead of a single +// (schema, table) pair. Kept as a separate literal rather than sharing a +// fragment — `pg_class` needs the `c` alias here for the oid filter, which +// pushes `information_schema.columns` to `ic` instead of the single-relation +// version's `c`, and every column's position in the SELECT list is relied on +// positionally by `column_info_from_row_offset`. A change to one almost +// certainly needs the same change in the other. fn postgres_columns_for_relations_sql() -> &'static str { "SELECT c.oid::bigint AS relid, a.attname AS column_name, \ format_type(a.atttypid, a.atttypmod) AS full_type, \ @@ -3352,6 +3380,8 @@ fn postgres_columns_for_relations_sql() -> &'static str { ORDER BY c.oid, a.attnum" } +// Compat-tier sibling of `POSTGRES_COLUMNS_COMPAT_SQL` (~line 4938) — see the +// note on `postgres_columns_for_relations_sql` above. fn postgres_columns_for_relations_compat_sql() -> &'static str { "SELECT c.oid::bigint AS relid, a.attname AS column_name, \ format_type(a.atttypid, a.atttypmod) AS full_type, \ @@ -3438,23 +3468,11 @@ pub async fn list_indexes_for_relations( ) -> Result>, String> { let oids: Vec = relations.iter().map(|(oid, _, _)| *oid).collect(); let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?; - let [primary_sql, compat_sql] = postgres_indexes_for_relations_query_tiers(); - match list_indexes_for_relations_with_sql(&client, primary_sql, &oids).await { - Ok(indexes) => Ok(indexes), - Err(primary_error) => match list_indexes_for_relations_with_sql(&client, compat_sql, &oids).await { - Ok(indexes) => Ok(indexes), - Err(fallback_error) => { - let primary_message = pg_error_to_string(primary_error); - let fallback_message = pg_error_to_string(fallback_error); - log::debug!( - "[postgres][list_indexes_for_relations:compat-failed] primary_error={} fallback_error={}", - primary_message, - fallback_message - ); - Err(fallback_message) - } - }, - } + let tiers = postgres_indexes_for_relations_query_tiers(); + query_with_compat_fallback("list_indexes_for_relations", &tiers, |sql| { + list_indexes_for_relations_with_sql(&client, sql, &oids) + }) + .await } async fn list_indexes_for_relations_with_sql( @@ -3493,6 +3511,11 @@ fn postgres_indexes_for_relations_query_tiers() -> [&'static str; 2] { [postgres_indexes_for_relations_sql(), postgres_indexes_for_relations_compat_sql()] } +// Sibling of `POSTGRES_INDEXES_SQL`/`POSTGRES_INDEXES_COMPAT_SQL` (~line +// 6042): same index-detection logic, batched by oid instead of a single +// (schema, table) pair. Not merged into a shared fragment for the same +// reason as the columns queries above — an alias would need renaming to +// line up, and result columns are read positionally. fn postgres_indexes_for_relations_sql() -> &'static str { "SELECT t.oid::bigint AS relid, i.relname AS index_name, \ array_agg(COALESCE(a.attname, pg_get_indexdef(ix.indexrelid, k.n::int, true)) ORDER BY k.n) AS columns, \ @@ -3515,6 +3538,8 @@ fn postgres_indexes_for_relations_sql() -> &'static str { ORDER BY t.oid, i.relname" } +// Compat-tier sibling of `POSTGRES_INDEXES_COMPAT_SQL` (~line 6063) — see the +// note on `postgres_indexes_for_relations_sql` above. fn postgres_indexes_for_relations_compat_sql() -> &'static str { "SELECT t.oid::bigint AS relid, i.relname AS index_name, \ ARRAY( \ @@ -5101,6 +5126,9 @@ pub async fn list_schema_infos_with_system(pool: &Pool, show_system_schemas: boo .collect()) } +// Sibling of `postgres_columns_for_relations_sql`/`_compat_sql` above (~line +// 3086), for a single (schema, table) instead of a batch of oids — see the +// note there about why these aren't merged into a shared fragment. const POSTGRES_COLUMNS_SQL: &str = "SELECT a.attname AS column_name, \ format_type(a.atttypid, a.atttypmod) AS full_type, \ COALESCE(c.is_nullable = 'YES', NOT a.attnotnull) AS is_nullable, \ @@ -5169,6 +5197,8 @@ const POSTGRES_COLUMNS_SQL: &str = "SELECT a.attname AS column_name, \ AND a.attnum > 0 AND NOT a.attisdropped \ ORDER BY a.attnum"; +// Compat-tier sibling of `postgres_columns_for_relations_compat_sql` (~line +// 3148) — see the note on `POSTGRES_COLUMNS_SQL` above. const POSTGRES_COLUMNS_COMPAT_SQL: &str = "SELECT a.attname AS column_name, \ format_type(a.atttypid, a.atttypmod) AS full_type, \ COALESCE(c.is_nullable = 'YES', NOT a.attnotnull) AS is_nullable, \ @@ -5348,29 +5378,8 @@ async fn get_columns_with_sql( pub async fn get_columns(pool: &Pool, schema: &str, table: &str) -> Result, String> { let schema = if schema.is_empty() { "public" } else { schema }; let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?; - match get_columns_with_sql(&client, POSTGRES_COLUMNS_SQL, schema, table).await { - Ok(columns) => Ok(columns), - Err(primary_error) => match get_columns_with_sql(&client, POSTGRES_COLUMNS_COMPAT_SQL, schema, table).await { - Ok(columns) => Ok(columns), - Err(fallback_error) => { - let primary_message = pg_error_to_string(primary_error); - let fallback_message = pg_error_to_string(fallback_error); - match get_columns_with_sql(&client, POSTGRES_COLUMNS_INFORMATION_SCHEMA_SQL, schema, table).await { - Ok(columns) => Ok(columns), - Err(information_schema_error) => { - let information_schema_message = pg_error_to_string(information_schema_error); - log::debug!( - "[postgres][get_columns:compat-failed] primary_error={} fallback_error={} information_schema_error={}", - primary_message, - fallback_message, - information_schema_message - ); - Err(information_schema_message) - } - } - } - }, - } + let tiers = [POSTGRES_COLUMNS_SQL, POSTGRES_COLUMNS_COMPAT_SQL, POSTGRES_COLUMNS_INFORMATION_SCHEMA_SQL]; + query_with_compat_fallback("get_columns", &tiers, |sql| get_columns_with_sql(&client, sql, schema, table)).await } fn pg_quote_literal(value: &str) -> String { @@ -6372,6 +6381,8 @@ async fn execute_query_with_max_rows_inner( } } +// Sibling of `postgres_indexes_for_relations_sql` (~line 3288), for a single +// (schema, table) instead of a batch of oids — see the note there. const POSTGRES_INDEXES_SQL: &str = "SELECT i.relname AS index_name, \ array_agg(COALESCE(a.attname, pg_get_indexdef(ix.indexrelid, k.n::int, true)) ORDER BY k.n) AS columns, \ ix.indisunique AS is_unique, \ @@ -6393,6 +6404,8 @@ const POSTGRES_INDEXES_SQL: &str = "SELECT i.relname AS index_name, \ GROUP BY i.relname, i.oid, ix.indisunique, ix.indisprimary, ix.indpred, ix.indrelid, am.amname, ix.indnkeyatts, ix.indkey \ ORDER BY i.relname"; +// Compat-tier sibling of `postgres_indexes_for_relations_compat_sql` (~line +// 3312) — see the note on `POSTGRES_INDEXES_SQL` above. const POSTGRES_INDEXES_COMPAT_SQL: &str = "SELECT i.relname AS index_name, \ ARRAY( \ SELECT COALESCE(a.attname, pg_get_indexdef(ix.indexrelid, pos.n, true)) \ @@ -6465,7 +6478,7 @@ const POSTGRES_COLUMN_ACL_PRIVILEGES_SQL: &str = WHERE n.nspname = $1 AND c.relname = $2 AND c.relkind IN ('r', 'p') \ ORDER BY 5, 1, 2, 3, 4"; -fn postgres_owner_object_type(relkind: &str) -> &str { +pub(crate) fn postgres_owner_object_type(relkind: &str) -> &str { match relkind { "r" => "TABLE", "v" => "VIEW", @@ -6478,6 +6491,29 @@ fn postgres_owner_object_type(relkind: &str) -> &str { } } +/// Looks up a relation's own `relkind` (e.g. `'r'`, `'v'`, `'S'`), regardless +/// of whether it's a kind this driver otherwise treats as a table. Used to +/// give a specific diagnostic ("it's a view, not a table") instead of a bare +/// "not found" when a `(schema, table)` request turns out not to be a +/// table/partition/foreign table. +pub(crate) async fn postgres_relation_relkind( + pool: &Pool, + schema: &str, + table: &str, +) -> Result, String> { + let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?; + let row = client + .query_opt( + "SELECT c.relkind::text FROM pg_catalog.pg_class c \ + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ + WHERE n.nspname = $1 AND c.relname = $2 LIMIT 1", + &[&schema, &table], + ) + .await + .map_err(pg_error_to_string)?; + row.map(|row| row.try_get::<_, String>(0)).transpose().map_err(pg_error_to_string) +} + async fn list_indexes_with_sql( client: &deadpool_postgres::Client, sql: &str, @@ -6516,22 +6552,8 @@ async fn list_indexes_with_sql( pub async fn list_indexes(pool: &Pool, schema: &str, table: &str) -> Result, String> { let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?; - match list_indexes_with_sql(&client, POSTGRES_INDEXES_SQL, schema, table).await { - Ok(indexes) => Ok(indexes), - Err(primary_error) => match list_indexes_with_sql(&client, POSTGRES_INDEXES_COMPAT_SQL, schema, table).await { - Ok(indexes) => Ok(indexes), - Err(fallback_error) => { - let primary_message = pg_error_to_string(primary_error); - let fallback_message = pg_error_to_string(fallback_error); - log::debug!( - "[postgres][list_indexes:compat-failed] primary_error={} fallback_error={}", - primary_message, - fallback_message - ); - Err(fallback_message) - } - }, - } + let tiers = [POSTGRES_INDEXES_SQL, POSTGRES_INDEXES_COMPAT_SQL]; + query_with_compat_fallback("list_indexes", &tiers, |sql| list_indexes_with_sql(&client, sql, schema, table)).await } /// Names of same-table indexes whose `pg_index.indisvalid` is `false`. diff --git a/crates/dbx-core/src/schema.rs b/crates/dbx-core/src/schema.rs index af70c3f790..185ca10e9f 100644 --- a/crates/dbx-core/src/schema.rs +++ b/crates/dbx-core/src/schema.rs @@ -9812,9 +9812,17 @@ pub async fn pg_ddl_with_partitions( let Some(root) = tree.iter().find(|node| !node.parent_oid.is_some_and(|parent_oid| tree_oids.contains(&parent_oid))) else { - return Err(format!( - "relation \"{schema}\".\"{table}\" was not found or is not a table/partition/foreign table" - )); + // The tree query only matches relkind IN ('r','p','f'), so an empty + // tree could mean the relation doesn't exist at all, or that it + // exists but is a view/sequence/other non-table object. Look up its + // relkind (if any) to tell those two cases apart in the error. + return Err(match db::postgres::postgres_relation_relkind(pool, schema, table).await { + Ok(Some(relkind)) => format!( + "relation \"{schema}\".\"{table}\" is a {}, not a table/partition/foreign table", + db::postgres::postgres_owner_object_type(&relkind) + ), + _ => format!("relation \"{schema}\".\"{table}\" was not found or is not a table/partition/foreign table"), + }); }; let oids: Vec = tree.iter().map(|node| node.oid).collect(); @@ -9841,6 +9849,18 @@ pub async fn pg_ddl_with_partitions( db::postgres::get_table_partition_local_objects_for_relations(pool, &oids), )?; + // `list_foreign_keys_for_relations` is keyed by (schema, table) since its + // query is information_schema-based, unlike every other *_by_oid map + // here. Remap it once so the render loop below can look fkeys up by oid + // like everything else, instead of cloning a fresh (schema, table) key + // on every node it renders. + let relation_oid_by_pair: HashMap<(String, String), i64> = + relations.into_iter().map(|(oid, schema, table)| ((schema, table), oid)).collect(); + let fkeys_by_oid: HashMap> = fkeys_by_relation + .into_iter() + .filter_map(|(key, fkeys)| relation_oid_by_pair.get(&key).map(|oid| (*oid, fkeys))) + .collect(); + // Group children by parent oid, each group ordered by relname to match // `list_table_partitions`' `ORDER BY c.relname` (today's traversal order). let mut children_by_parent: HashMap> = HashMap::new(); @@ -9859,7 +9879,7 @@ pub async fn pg_ddl_with_partitions( &children_by_parent, &columns_by_oid, &indexes_by_oid, - &fkeys_by_relation, + &fkeys_by_oid, &comments_by_oid, &triggers_by_oid, &checks_by_oid, @@ -9880,7 +9900,7 @@ fn render_postgres_partition_tree_node( children_by_parent: &HashMap>, columns_by_oid: &HashMap>, indexes_by_oid: &HashMap>, - fkeys_by_relation: &HashMap<(String, String), Vec>, + fkeys_by_oid: &HashMap>, comments_by_oid: &HashMap>, triggers_by_oid: &HashMap>, checks_by_oid: &HashMap>, @@ -9903,8 +9923,8 @@ fn render_postgres_partition_tree_node( let columns = columns_by_oid.get(&node.oid).unwrap_or(&empty_columns); let indexes = indexes_by_oid.get(&node.oid).unwrap_or(&empty_indexes); - let fkeys = fkeys_by_relation.get(&(node.schema.clone(), node.table.clone())).unwrap_or(&empty_fkeys); - let comment = comments_by_oid.get(&node.oid).cloned().flatten(); + let fkeys = fkeys_by_oid.get(&node.oid).unwrap_or(&empty_fkeys); + let comment = comments_by_oid.get(&node.oid).and_then(|comment| comment.as_deref()); let triggers = triggers_by_oid.get(&node.oid).unwrap_or(&empty_triggers); let checks = checks_by_oid.get(&node.oid).unwrap_or(&empty_checks); let local_objects = local_objects_by_oid.get(&node.oid).unwrap_or(&empty_local_objects); @@ -9920,7 +9940,7 @@ fn render_postgres_partition_tree_node( indexes, fkeys, checks, - comment.as_deref(), + comment, &node.partition_info, local_objects, ), @@ -10195,13 +10215,82 @@ fn append_opengauss_trigger_definitions(mut ddl: String, trigger_definitions: &[ ddl } +/// Cloudberry's native `pg_get_tabledef` (a community-maintained PL/pgSQL +/// function most Cloudberry/Greenplum installs have, not a built-in) renders +/// exactly one relation — for a partitioned table it emits that table's own +/// `CREATE TABLE ... PARTITION BY ...`, never its partitions' own `CREATE +/// TABLE ... PARTITION OF ...` statements, regardless of whether the table +/// was created with classic Greenplum or PostgreSQL-style declarative +/// partition syntax. When `include_partitions` is requested, fetch the same +/// partition tree the plain-Postgres path uses and append each descendant's +/// own native DDL, so Cloudberry's more accurate native rendering (storage +/// options, distribution policy, external-table clauses) is still used per +/// relation instead of falling back to the generic renderer for the whole +/// tree. +async fn cloudberry_native_tree_ddl( + pool: &deadpool_postgres::Pool, + schema: &str, + table: &str, + include_partitions: bool, +) -> Result { + let root_ddl = db::cloudberry::table_ddl(pool, schema, table).await?; + if !include_partitions { + return Ok(root_ddl); + } + let tree = db::postgres::fetch_postgres_partition_tree(pool, schema, table).await?; + if tree.len() <= 1 { + return Ok(root_ddl); + } + let tree_oids: HashSet = tree.iter().map(|node| node.oid).collect(); + let Some(root) = + tree.iter().find(|node| !node.parent_oid.is_some_and(|parent_oid| tree_oids.contains(&parent_oid))) + else { + return Ok(root_ddl); + }; + + let mut children_by_parent: HashMap> = HashMap::new(); + for node in &tree { + if let Some(parent_oid) = node.parent_oid { + children_by_parent.entry(parent_oid).or_default().push(node); + } + } + for children in children_by_parent.values_mut() { + children.sort_by(|a, b| a.table.cmp(&b.table)); + } + + let mut descendants = Vec::new(); + let mut visited: HashSet = HashSet::new(); + let mut stack: Vec<&db::postgres::PostgresPartitionTreeNode> = vec![root]; + while let Some(node) = stack.pop() { + if !visited.insert(node.oid) { + continue; + } + if node.oid != root.oid { + descendants.push(node); + } + if let Some(children) = children_by_parent.get(&node.oid) { + for child in children.iter().rev() { + stack.push(child); + } + } + } + + let mut ddl = root_ddl; + for node in descendants { + let child_ddl = db::cloudberry::table_ddl(pool, &node.schema, &node.table).await?; + ddl.push('\n'); + ddl.push_str(&child_ddl); + } + Ok(ddl) +} + pub async fn cloudberry_ddl( pool: &deadpool_postgres::Pool, schema: &str, table: &str, include_partitions: bool, ) -> Result { - match db::cloudberry::table_ddl(pool, schema, table).await { + match cloudberry_native_tree_ddl(pool, schema, table, include_partitions).await { Ok(ddl) => Ok(ddl), Err(native_error) => { let base_ddl = pg_ddl_for_options(pool, schema, table, include_partitions).await.map_err(|fallback_error| { @@ -10228,6 +10317,39 @@ pub async fn opentenbase_ddl( include_partitions: bool, ) -> Result { let ddl = pg_ddl_for_options(pool, schema, table, include_partitions).await?; + if include_partitions { + // The rendered ddl may cover the whole partition tree (root plus + // every partition), each as its own `CREATE [FOREIGN] TABLE` + // statement, so distribution policies need to be looked up and + // applied per relation rather than once for the root. + let relations: Vec<(String, String)> = db::ddl_scan::top_level_statement_ranges(&ddl) + .into_iter() + .filter_map(|range| db::ddl_scan::parse_create_table_relation(&ddl[range])) + .collect(); + return match db::opentenbase::table_distribution_for_relations(pool, &relations).await { + Ok(distributions) => match db::opentenbase::append_distribution_clauses(&ddl, &distributions) { + Ok(ddl) => Ok(ddl), + Err(error) => { + log::warn!( + "[schema][opentenbase:table-ddl-distribution-render-fallback] schema={} table={} error={}", + schema, + table, + error + ); + Ok(ddl) + } + }, + Err(error) => { + log::warn!( + "[schema][opentenbase:table-ddl-distribution-query-fallback] schema={} table={} error={}", + schema, + table, + error + ); + Ok(ddl) + } + }; + } match db::opentenbase::table_distribution(pool, schema, table).await { Ok(Some(distribution)) => match db::opentenbase::append_distribution_clause(&ddl, &distribution) { Ok(ddl) => Ok(ddl), diff --git a/crates/dbx-core/tests/database_export_partition_ddl.rs b/crates/dbx-core/tests/database_export_partition_ddl.rs index 7ef306334d..6cf0e758aa 100644 --- a/crates/dbx-core/tests/database_export_partition_ddl.rs +++ b/crates/dbx-core/tests/database_export_partition_ddl.rs @@ -10,164 +10,17 @@ //! number of partitions. //! Needs local Docker; skips cleanly when unavailable. +mod support; + use std::process::Command; use dbx_core::connection::AppState; use dbx_core::database_export::{export_database_sql_core, DatabaseExportRequest}; -use dbx_core::models::connection::{ConnectionConfig, DatabaseType}; use dbx_core::storage::Storage; - -struct DockerPostgres { - name: String, - port: u16, - remove_on_drop: bool, -} - -impl Drop for DockerPostgres { - fn drop(&mut self) { - if self.remove_on_drop { - let _ = Command::new("docker").args(["rm", "-f", &self.name]).status(); - } - } -} - -fn docker_ready() -> bool { - Command::new("docker") - .args(["version", "--format", "{{.Server.Version}}"]) - .output() - .map(|output| output.status.success()) - .unwrap_or(false) -} - -fn start_docker_postgres(name_prefix: &str) -> Option { - if let Ok(name) = std::env::var("DBX_TEST_POSTGRES_CONTAINER") { - let port = std::env::var("DBX_TEST_POSTGRES_PORT") - .expect("DBX_TEST_POSTGRES_PORT must accompany DBX_TEST_POSTGRES_CONTAINER") - .parse() - .expect("DBX_TEST_POSTGRES_PORT must be a valid port"); - return Some(DockerPostgres { name, port, remove_on_drop: false }); - } - if !docker_ready() { - eprintln!("skipping docker-backed partition ddl test because Docker is unavailable"); - return None; - } - - let port = portpicker::pick_unused_port().expect("pick unused postgres port"); - let container = - DockerPostgres { name: format!("{name_prefix}-{}", uuid::Uuid::new_v4()), port, remove_on_drop: true }; - - let status = Command::new("docker") - .args([ - "run", - "-d", - "--rm", - "--name", - &container.name, - "-e", - "POSTGRES_PASSWORD=postgres", - "-e", - "POSTGRES_USER=postgres", - "-e", - "POSTGRES_DB=postgres", - "-p", - &format!("{port}:5432"), - "postgres:16-alpine", - ]) - .status() - .expect("start docker postgres"); - assert!(status.success(), "docker run postgres container should succeed"); - - let mut consecutive_ok = 0; - for _ in 0..120 { - let ready = Command::new("docker") - .args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-c", "SELECT 1"]) - .output() - .map(|output| output.status.success()) - .unwrap_or(false); - consecutive_ok = if ready { consecutive_ok + 1 } else { 0 }; - if consecutive_ok >= 2 { - return Some(container); - } - std::thread::sleep(std::time::Duration::from_millis(500)); - } - panic!("postgres container did not become ready"); -} - -fn psql(container: &DockerPostgres, sql: &str) { - let output = Command::new("docker") - .args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-c", sql]) - .output() - .expect("run psql"); - assert!(output.status.success(), "psql failed: {}", String::from_utf8_lossy(&output.stderr)); -} - -fn psql_allow_failure(container: &DockerPostgres, sql: &str) -> bool { - Command::new("docker") - .args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-c", sql]) - .output() - .map(|output| output.status.success()) - .unwrap_or(false) -} - -fn postgres_test_config(id: &str, port: u16) -> ConnectionConfig { - ConnectionConfig { - docs_notes_path: None, - id: id.to_string(), - name: id.to_string(), - note: String::new(), - db_type: DatabaseType::Postgres, - driver_profile: None, - driver_label: None, - url_params: None, - agent_java_options: Vec::new(), - host: "127.0.0.1".to_string(), - port, - username: "postgres".to_string(), - password: "postgres".to_string(), - database: Some("postgres".to_string()), - default_schema: None, - visible_databases: None, - visible_schemas: None, - attached_databases: Vec::new(), - init_script: None, - color: None, - transport_layers: Vec::new(), - connect_timeout_secs: 5, - query_timeout_secs: 30, - idle_timeout_secs: 60, - keepalive_interval_secs: 0, - ssl: false, - ca_cert_path: String::new(), - client_cert_path: String::new(), - client_key_path: String::new(), - sysdba: false, - oracle_connection_type: None, - connection_string: None, - redis_connection_mode: None, - redis_sentinel_master: String::new(), - redis_sentinel_nodes: String::new(), - redis_sentinel_username: String::new(), - redis_sentinel_password: String::new(), - redis_sentinel_tls: false, - redis_cluster_nodes: String::new(), - redis_key_separator: dbx_core::models::connection::default_redis_key_separator(), - redis_scan_page_size: None, - redis_database_aliases: Default::default(), - etcd_endpoints: String::new(), - gbase_server: String::new(), - informix_server: String::new(), - external_config: None, - jdbc_driver_class: None, - jdbc_driver_paths: Vec::new(), - one_time: false, - save_password: true, - read_only: false, - is_production: false, - production_databases: vec![], - show_system_schemas: false, - database_info: None, - } -} +use support::{ + postgres_test_config, psql, psql_allow_failure, start_docker_postgres, start_docker_postgres_with_server_args, + DockerPostgres, +}; /// Full database export of a partitioned schema must not duplicate any /// relation's `CREATE TABLE`/`CREATE FOREIGN TABLE`, must faithfully replay @@ -433,3 +286,156 @@ async fn partition_tree_ddl_query_count_does_not_scale_with_partition_count() { "query count should not scale with partition count: small={small_count} large={large_count}" ); } + +/// A tree-DDL request for a relation that exists but isn't a +/// table/partition/foreign table (e.g. a view) must say so specifically, +/// not fall back to a generic "not found" message that also covers a +/// relation that doesn't exist at all. +#[tokio::test] +async fn display_ddl_error_distinguishes_wrong_relkind_from_missing_relation() { + let Some(container) = start_docker_postgres("dbx-partition-ddl-relkind-error") else { + return; + }; + + psql(&container, "CREATE VIEW v AS SELECT 1 AS id"); + + let dir = std::env::temp_dir().join(format!("dbx-partition-relkind-error-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let storage = Storage::open(&dir.join("storage.db")).await.unwrap(); + let state = AppState::new(storage); + let connection_id = "partition-relkind-error-conn"; + state.configs.write().await.insert(connection_id.to_string(), postgres_test_config(connection_id, container.port)); + + let view_error = + dbx_core::schema::get_table_display_ddl_core(&state, connection_id, "postgres", "public", "v", None) + .await + .expect_err("requesting tree DDL for a view must fail"); + assert!(view_error.contains("is a VIEW"), "expected a relkind-specific error, got: {view_error}"); + + let missing_error = dbx_core::schema::get_table_display_ddl_core( + &state, + connection_id, + "postgres", + "public", + "does_not_exist", + None, + ) + .await + .expect_err("requesting tree DDL for a missing relation must fail"); + assert!(missing_error.contains("was not found"), "expected a not-found error, got: {missing_error}"); + assert!(!missing_error.contains("is a"), "not-found error should not claim a relkind: {missing_error}"); +} + +/// Cloudberry/Greenplum's `pg_get_tabledef` (a community PL/pgSQL function +/// most installs have, not a catalog builtin) renders exactly the one +/// relation it's asked for — never its partitions' own `CREATE TABLE ... +/// PARTITION OF ...` statements. `cloudberry_ddl(..., include_partitions: +/// true)` must still return every partition's DDL by walking the same +/// partition tree the plain-Postgres path uses, instead of silently +/// returning just the root's DDL whenever the native call succeeds. This +/// installs a minimal stand-in `pg_get_tabledef` (mirroring the real one's +/// single-relation behavior) rather than the full upstream script, since +/// only that single-relation behavior is what the fix under test reacts to. +#[tokio::test] +async fn cloudberry_native_ddl_still_includes_every_partition() { + let Some(container) = start_docker_postgres("dbx-cloudberry-partition-ddl") else { + return; + }; + + psql( + &container, + "CREATE OR REPLACE FUNCTION pg_get_tabledef(in_schema text, in_table text, _verbose boolean) \ + RETURNS text AS $$ SELECT format('CREATE TABLE \"%s\".\"%s\" (id integer) /* stub-native-ddl */;', in_schema, in_table) $$ \ + LANGUAGE sql", + ); + psql(&container, "CREATE TABLE t (id integer NOT NULL) PARTITION BY RANGE (id)"); + psql(&container, "CREATE TABLE t_p1 PARTITION OF t FOR VALUES FROM (0) TO (100)"); + psql(&container, "CREATE TABLE t_p2 PARTITION OF t FOR VALUES FROM (100) TO (200)"); + + let url = format!("postgres://postgres:postgres@127.0.0.1:{}/postgres", container.port); + let pool = dbx_core::db::postgres::connect(&url, std::time::Duration::from_secs(5)).await.expect("connect"); + + let ddl = + dbx_core::schema::cloudberry_ddl(&pool, "public", "t", true).await.expect("cloudberry ddl with partitions"); + assert_eq!( + ddl.matches("stub-native-ddl").count(), + 3, + "expected the root plus both partitions to each get their own native DDL: {ddl}" + ); + assert!(ddl.contains("CREATE TABLE \"public\".\"t\"")); + assert!(ddl.contains("CREATE TABLE \"public\".\"t_p1\"")); + assert!(ddl.contains("CREATE TABLE \"public\".\"t_p2\"")); + + let single_ddl = + dbx_core::schema::cloudberry_ddl(&pool, "public", "t", false).await.expect("cloudberry ddl without partitions"); + assert_eq!(single_ddl.matches("stub-native-ddl").count(), 1, "expected only the root's own DDL: {single_ddl}"); +} + +/// `opentenbase_ddl(..., include_partitions: true)` must query and append +/// each partition's own `DISTRIBUTE BY` clause, not just the tree root's. +/// OpenTenBase itself has no simple single-container way to stand up (its +/// own docs only cover a multi-node `pgxc_ctl deploy`/`init` cluster built +/// from source), so this runs the actual, unmodified +/// `TABLE_DISTRIBUTION_FOR_RELATIONS_SQL` query against a real PostgreSQL +/// container with a stand-in `pg_catalog.pgxc_class` table shaped like +/// OpenTenBase's real one — exercising the real SQL (join, `unnest($1:: +/// text[], $2::text[])` parameter binding, casts, row parsing), not just the +/// clause-insertion string logic around it. +#[tokio::test] +async fn opentenbase_tree_ddl_gives_every_partition_its_own_distribute_by() { + let Some(container) = + start_docker_postgres_with_server_args("dbx-opentenbase-partition-ddl", &["-c", "allow_system_table_mods=on"]) + else { + return; + }; + + psql( + &container, + "CREATE TABLE pg_catalog.pgxc_class (pcrelid oid PRIMARY KEY, pclocatortype \"char\", discolnums smallint[])", + ); + psql(&container, "CREATE TABLE t (id integer NOT NULL, region text NOT NULL) PARTITION BY RANGE (id)"); + psql(&container, "CREATE TABLE t_p1 PARTITION OF t FOR VALUES FROM (0) TO (100)"); + psql(&container, "CREATE TABLE t_p2 PARTITION OF t FOR VALUES FROM (100) TO (200)"); + // Root distributes by `region` (attnum 2); t_p1 distributes by `id` + // (attnum 1) instead, to prove each relation's own policy is looked up + // independently rather than the root's being reused for every relation. + // t_p2 deliberately has no pgxc_class row at all — it must be left + // without a DISTRIBUTE BY clause rather than erroring the whole DDL. + psql(&container, "INSERT INTO pg_catalog.pgxc_class VALUES ('t'::regclass, 'H', ARRAY[2]::smallint[])"); + psql(&container, "INSERT INTO pg_catalog.pgxc_class VALUES ('t_p1'::regclass, 'H', ARRAY[1]::smallint[])"); + + let url = format!("postgres://postgres:postgres@127.0.0.1:{}/postgres", container.port); + let pool = dbx_core::db::postgres::connect(&url, std::time::Duration::from_secs(5)).await.expect("connect"); + + let distributions = dbx_core::db::opentenbase::table_distribution_for_relations( + &pool, + &[("public".to_string(), "t".to_string()), ("public".to_string(), "t_p1".to_string())], + ) + .await + .expect("batched distribution query"); + assert_eq!( + distributions.get(&("public".to_string(), "t".to_string())), + Some(&dbx_core::db::opentenbase::DistributionPolicy::Hash(vec!["region".to_string()])) + ); + assert_eq!( + distributions.get(&("public".to_string(), "t_p1".to_string())), + Some(&dbx_core::db::opentenbase::DistributionPolicy::Hash(vec!["id".to_string()])) + ); + + let ddl = dbx_core::schema::opentenbase_ddl(&pool, "public", "t", true).await.expect("opentenbase tree ddl"); + assert!( + ddl.contains("\"t\" (\n \"id\" integer NOT NULL,\n \"region\" text NOT NULL\n) PARTITION BY RANGE (id)\nDISTRIBUTE BY HASH (\"region\");"), + "root should get its own DISTRIBUTE BY HASH (region): {ddl}" + ); + assert!( + ddl.contains( + "\"t_p1\" PARTITION OF \"public\".\"t\" FOR VALUES FROM (0) TO (100)\nDISTRIBUTE BY HASH (\"id\");" + ), + "t_p1 should get its own DISTRIBUTE BY HASH (id), independent of the root's: {ddl}" + ); + assert!( + ddl.contains("\"t_p2\" PARTITION OF \"public\".\"t\" FOR VALUES FROM (100) TO (200);"), + "t_p2 has no pgxc_class row and must not get a DISTRIBUTE BY clause: {ddl}" + ); + assert_eq!(ddl.matches("DISTRIBUTE BY").count(), 2); +} diff --git a/crates/dbx-core/tests/database_export_prefetch.rs b/crates/dbx-core/tests/database_export_prefetch.rs index c02bbf3fa9..07310c0a42 100644 --- a/crates/dbx-core/tests/database_export_prefetch.rs +++ b/crates/dbx-core/tests/database_export_prefetch.rs @@ -1,147 +1,12 @@ //! 整库 SQL 导出的端到端回归测试(元数据并发预取后输出必须与逐表查询一致)。 //! 需要本机 Docker;不可用时自动跳过。 -use std::process::Command; +mod support; use dbx_core::connection::AppState; use dbx_core::database_export::{export_database_sql_core, DatabaseExportRequest}; -use dbx_core::models::connection::{ConnectionConfig, DatabaseType}; use dbx_core::storage::Storage; - -struct DockerPostgres { - name: String, - port: u16, -} - -impl Drop for DockerPostgres { - fn drop(&mut self) { - let _ = Command::new("docker").args(["rm", "-f", &self.name]).status(); - } -} - -fn docker_ready() -> bool { - Command::new("docker") - .args(["version", "--format", "{{.Server.Version}}"]) - .output() - .map(|output| output.status.success()) - .unwrap_or(false) -} - -fn start_docker_postgres() -> Option { - if !docker_ready() { - eprintln!("skipping docker-backed export test because Docker is unavailable"); - return None; - } - - let port = portpicker::pick_unused_port().expect("pick unused postgres port"); - let container = DockerPostgres { name: format!("dbx-export-prefetch-{}", uuid::Uuid::new_v4()), port }; - - let status = Command::new("docker") - .args([ - "run", - "-d", - "--rm", - "--name", - &container.name, - "-e", - "POSTGRES_PASSWORD=postgres", - "-e", - "POSTGRES_USER=postgres", - "-e", - "POSTGRES_DB=postgres", - "-p", - &format!("{port}:5432"), - "postgres:16-alpine", - ]) - .status() - .expect("start docker postgres"); - assert!(status.success(), "docker run postgres container should succeed"); - - // postgres 镜像 initdb 期间会短暂拉起再重启服务,单次探测可能命中引导期, - // 要求 SELECT 1 连续两次成功才算就绪 - let mut consecutive_ok = 0; - for _ in 0..120 { - let ready = Command::new("docker") - .args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-c", "SELECT 1"]) - .output() - .map(|output| output.status.success()) - .unwrap_or(false); - consecutive_ok = if ready { consecutive_ok + 1 } else { 0 }; - if consecutive_ok >= 2 { - return Some(container); - } - std::thread::sleep(std::time::Duration::from_millis(500)); - } - panic!("postgres container did not become ready"); -} - -fn psql(container: &DockerPostgres, sql: &str) { - let output = Command::new("docker") - .args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-c", sql]) - .output() - .expect("run psql"); - assert!(output.status.success(), "psql failed: {}", String::from_utf8_lossy(&output.stderr)); -} - -fn postgres_test_config(id: &str, port: u16) -> ConnectionConfig { - ConnectionConfig { - docs_notes_path: None, - id: id.to_string(), - name: id.to_string(), - note: String::new(), - db_type: DatabaseType::Postgres, - driver_profile: None, - driver_label: None, - url_params: None, - agent_java_options: Vec::new(), - host: "127.0.0.1".to_string(), - port, - username: "postgres".to_string(), - password: "postgres".to_string(), - database: Some("postgres".to_string()), - default_schema: None, - visible_databases: None, - visible_schemas: None, - attached_databases: Vec::new(), - init_script: None, - color: None, - transport_layers: Vec::new(), - connect_timeout_secs: 5, - query_timeout_secs: 30, - idle_timeout_secs: 60, - keepalive_interval_secs: 0, - ssl: false, - ca_cert_path: String::new(), - client_cert_path: String::new(), - client_key_path: String::new(), - sysdba: false, - oracle_connection_type: None, - connection_string: None, - redis_connection_mode: None, - redis_sentinel_master: String::new(), - redis_sentinel_nodes: String::new(), - redis_sentinel_username: String::new(), - redis_sentinel_password: String::new(), - redis_sentinel_tls: false, - redis_cluster_nodes: String::new(), - redis_key_separator: dbx_core::models::connection::default_redis_key_separator(), - redis_scan_page_size: None, - redis_database_aliases: Default::default(), - etcd_endpoints: String::new(), - gbase_server: String::new(), - informix_server: String::new(), - external_config: None, - jdbc_driver_class: None, - jdbc_driver_paths: Vec::new(), - one_time: false, - save_password: true, - read_only: false, - is_production: false, - production_databases: vec![], - show_system_schemas: false, - database_info: None, - } -} +use support::{postgres_test_config, psql, start_docker_postgres}; #[test] fn database_export_writes_structure_and_data_for_all_tables() { @@ -162,7 +27,7 @@ fn database_export_writes_structure_and_data_for_all_tables() { } async fn run_database_export_writes_structure_and_data_for_all_tables() { - let Some(container) = start_docker_postgres() else { + let Some(container) = start_docker_postgres("dbx-export-prefetch") else { return; }; diff --git a/crates/dbx-core/tests/support/mod.rs b/crates/dbx-core/tests/support/mod.rs new file mode 100644 index 0000000000..d3434f1d28 --- /dev/null +++ b/crates/dbx-core/tests/support/mod.rs @@ -0,0 +1,191 @@ +//! Shared Docker-backed PostgreSQL test scaffolding for integration tests +//! that need a real, disposable server (not a mock) to exercise driver SQL +//! against. Needs local Docker; callers should check `docker_ready()` (via +//! `start_docker_postgres`, which does this for them) and skip cleanly when +//! it's unavailable rather than failing the whole test run. + +use std::process::Command; + +use dbx_core::models::connection::{ConnectionConfig, DatabaseType}; + +pub struct DockerPostgres { + pub name: String, + pub port: u16, + /// `false` when this wraps an already-running container the caller + /// provided via `DBX_TEST_POSTGRES_CONTAINER` (see + /// `start_docker_postgres_with_server_args`) instead of one this helper + /// started itself — that container is the caller's to manage, not ours + /// to tear down. + remove_on_drop: bool, +} + +impl Drop for DockerPostgres { + fn drop(&mut self) { + if self.remove_on_drop { + let _ = Command::new("docker").args(["rm", "-f", &self.name]).status(); + } + } +} + +pub fn docker_ready() -> bool { + Command::new("docker") + .args(["version", "--format", "{{.Server.Version}}"]) + .output() + .map(|output| output.status.success()) + .unwrap_or(false) +} + +/// Starts a disposable `postgres:16-alpine` container named +/// `{name_prefix}-{random uuid}`, waiting for it to accept connections. +/// Returns `None` (rather than panicking) when Docker itself isn't +/// available, so tests can skip cleanly in environments without it. +pub fn start_docker_postgres(name_prefix: &str) -> Option { + start_docker_postgres_with_server_args(name_prefix, &[]) +} + +/// Same as `start_docker_postgres`, but forwards extra `-c name=value` +/// postmaster arguments — e.g. `&["-c", "allow_system_table_mods=on"]` to let +/// a test create a stand-in `pg_catalog` table for a system catalog this +/// image doesn't actually have (like OpenTenBase/Cloudberry-only ones). +/// +/// If `DBX_TEST_POSTGRES_CONTAINER`/`DBX_TEST_POSTGRES_PORT` are set, reuses +/// that already-running container instead of starting (and later tearing +/// down) a fresh one — handy for iterating on a single test locally without +/// paying container startup cost every run. `server_args` is ignored in that +/// case, since the caller's container is already configured however they set +/// it up. +pub fn start_docker_postgres_with_server_args(name_prefix: &str, server_args: &[&str]) -> Option { + if let Ok(name) = std::env::var("DBX_TEST_POSTGRES_CONTAINER") { + let port = std::env::var("DBX_TEST_POSTGRES_PORT") + .expect("DBX_TEST_POSTGRES_PORT must accompany DBX_TEST_POSTGRES_CONTAINER") + .parse() + .expect("DBX_TEST_POSTGRES_PORT must be a valid port"); + return Some(DockerPostgres { name, port, remove_on_drop: false }); + } + if !docker_ready() { + eprintln!("skipping docker-backed test because Docker is unavailable"); + return None; + } + + let port = portpicker::pick_unused_port().expect("pick unused postgres port"); + let container = + DockerPostgres { name: format!("{name_prefix}-{}", uuid::Uuid::new_v4()), port, remove_on_drop: true }; + + let mut args = vec![ + "run".to_string(), + "-d".to_string(), + "--rm".to_string(), + "--name".to_string(), + container.name.clone(), + "-e".to_string(), + "POSTGRES_PASSWORD=postgres".to_string(), + "-e".to_string(), + "POSTGRES_USER=postgres".to_string(), + "-e".to_string(), + "POSTGRES_DB=postgres".to_string(), + "-p".to_string(), + format!("{port}:5432"), + "postgres:16-alpine".to_string(), + ]; + args.extend(server_args.iter().map(|arg| arg.to_string())); + + let status = Command::new("docker").args(&args).status().expect("start docker postgres"); + assert!(status.success(), "docker run postgres container should succeed"); + + // The postgres image's initdb briefly starts and restarts the server, so + // a single successful probe may still land inside that bootstrap window; + // require two consecutive successful `SELECT 1`s before calling it ready. + let mut consecutive_ok = 0; + for _ in 0..120 { + let ready = Command::new("docker") + .args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-c", "SELECT 1"]) + .output() + .map(|output| output.status.success()) + .unwrap_or(false); + consecutive_ok = if ready { consecutive_ok + 1 } else { 0 }; + if consecutive_ok >= 2 { + return Some(container); + } + std::thread::sleep(std::time::Duration::from_millis(500)); + } + panic!("postgres container did not become ready"); +} + +pub fn psql(container: &DockerPostgres, sql: &str) { + let output = Command::new("docker") + .args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-c", sql]) + .output() + .expect("run psql"); + assert!(output.status.success(), "psql failed: {}", String::from_utf8_lossy(&output.stderr)); +} + +// Each `tests/*.rs` file compiles this module as its own separate crate, so +// a helper only some of those test binaries call is "unused" in the others. +#[allow(dead_code)] +pub fn psql_allow_failure(container: &DockerPostgres, sql: &str) -> bool { + Command::new("docker") + .args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-c", sql]) + .output() + .map(|output| output.status.success()) + .unwrap_or(false) +} + +pub fn postgres_test_config(id: &str, port: u16) -> ConnectionConfig { + ConnectionConfig { + docs_notes_path: None, + id: id.to_string(), + name: id.to_string(), + note: String::new(), + db_type: DatabaseType::Postgres, + driver_profile: None, + driver_label: None, + url_params: None, + agent_java_options: Vec::new(), + host: "127.0.0.1".to_string(), + port, + username: "postgres".to_string(), + password: "postgres".to_string(), + database: Some("postgres".to_string()), + default_schema: None, + visible_databases: None, + visible_schemas: None, + attached_databases: Vec::new(), + init_script: None, + color: None, + transport_layers: Vec::new(), + connect_timeout_secs: 5, + query_timeout_secs: 30, + idle_timeout_secs: 60, + keepalive_interval_secs: 0, + ssl: false, + ca_cert_path: String::new(), + client_cert_path: String::new(), + client_key_path: String::new(), + sysdba: false, + oracle_connection_type: None, + connection_string: None, + redis_connection_mode: None, + redis_sentinel_master: String::new(), + redis_sentinel_nodes: String::new(), + redis_sentinel_username: String::new(), + redis_sentinel_password: String::new(), + redis_sentinel_tls: false, + redis_cluster_nodes: String::new(), + redis_key_separator: dbx_core::models::connection::default_redis_key_separator(), + redis_scan_page_size: None, + redis_database_aliases: Default::default(), + etcd_endpoints: String::new(), + gbase_server: String::new(), + informix_server: String::new(), + external_config: None, + jdbc_driver_class: None, + jdbc_driver_paths: Vec::new(), + one_time: false, + save_password: true, + read_only: false, + is_production: false, + production_databases: vec![], + show_system_schemas: false, + database_info: None, + } +}