fix(postgres): correct partition DDL generation

This commit is contained in:
Melody Ruch authored and GitHub committed 2026-08-19 15:08:54 +08:00
1 parent e9ca697c3b
commit d085c5b4f4
9 files changed
+960 -378

No files matched your search

+36 -4
View File
@@ -170,10 +170,14 @@ fn render_external_table_ddl(ddl: &str, external: &ExternalTableDefinition) -> R
.ok_or_else(|| "Cloudberry fallback DDL has no CREATE TABLE terminator".to_string())?;
// When the fallback DDL already read `CREATE FOREIGN TABLE`, the generic
// renderer also already appended its own ` SERVER ... [OPTIONS (...)]`
// clause (from the same relkind='f' catalog data). Drop that so this
// doesn't emit the clause twice — Cloudberry's own `external` definition
// above is authoritative here.
let body_end = ddl[header_len..insertion].find(" SERVER \"").map(|pos| header_len + pos).unwrap_or(insertion);
// clause (from the same relkind='f' catalog data) right after the column
// list's closing paren. Drop that so this doesn't emit the clause twice —
// Cloudberry's own `external` definition above is authoritative here.
// Locate the column list's own closing paren structurally (tracking
// parenthesis depth and skipping quoted content) instead of searching for
// the literal text `" SERVER \""`, which a column's DEFAULT/CHECK
// expression could contain and match spuriously.
let body_end = super::ddl_scan::find_top_level_paren_close(ddl, header_len).unwrap_or(insertion);
let mut output = String::with_capacity(ddl.len() + external.options.len() * 24 + 48);
output.push_str("CREATE FOREIGN TABLE ");
output.push_str(&ddl[header_len..body_end]);
@@ -364,6 +368,34 @@ mod tests {
assert!(!rendered.contains("stale_server"));
}
#[test]
fn renders_external_table_when_a_column_default_contains_literal_server_text() {
// A naive text search for `" SERVER \""` to find the stale SERVER
// clause's start would match this DEFAULT's embedded text instead of
// the real trailing SERVER clause, truncating the column list.
let ddl = "CREATE FOREIGN TABLE \"public\".\"external_events\" (\n \"id\" integer,\n \"note\" text DEFAULT 'contact SERVER \"admin\"'\n) SERVER \"stale_server\";\n";
let rendered = append_table_modifiers(
ddl,
&TableModifiers {
external: Some(ExternalTableDefinition {
server: "gp_exttable_server".to_string(),
options: vec!["format=csv".to_string()],
}),
..modifiers(None)
},
)
.unwrap();
assert!(rendered.contains("\"note\" text DEFAULT 'contact SERVER \"admin\"'"));
assert_eq!(
rendered.matches("SERVER").count(),
2,
"expected exactly the embedded literal plus the real clause: {rendered}"
);
assert!(rendered.contains("SERVER \"gp_exttable_server\""));
assert!(!rendered.contains("stale_server"));
}
#[test]
fn ddl_queries_avoid_privileged_external_catalog_view() {
assert_eq!(CLOUD_BERRY_TABLE_DDL_SQL, "SELECT pg_get_tabledef($1, $2, true)");
+223
View File
@@ -0,0 +1,223 @@
//! Structural, quote-and-parenthesis-aware scanning helpers for
//! post-processing already-rendered `CREATE TABLE` DDL text.
//!
//! A naive `str::find` for a literal marker (e.g. `" SERVER \""` or a bare
//! `;`) can be fooled by that same text appearing inside a quoted string
//! literal, a quoted identifier, or a nested parenthesized sub-expression —
//! a column's `DEFAULT`/`CHECK` expression can legally contain any of those.
//! These helpers track quote and parenthesis depth so callers only ever
//! match at the top level.
use std::ops::Range;
fn char_at(sql: &str, pos: usize) -> Option<char> {
sql.get(pos..)?.chars().next()
}
/// Skips a `'...'` string literal (`''`-escaped, with backslash-escape
/// tolerance) or a `"..."` quoted identifier (`""`-escaped) starting at
/// `pos`, which must point at the opening `quote` character. Returns the
/// index just past the closing quote (or `sql.len()` if unterminated).
fn skip_quoted(sql: &str, pos: usize, quote: char) -> usize {
let mut i = pos + quote.len_utf8();
while i < sql.len() {
let Some(ch) = char_at(sql, i) else { break };
let next = char_at(sql, i + ch.len_utf8());
if ch == quote {
if next == Some(quote) {
i += ch.len_utf8() + quote.len_utf8();
continue;
}
return i + ch.len_utf8();
}
if quote == '\'' && ch == '\\' {
i += ch.len_utf8();
if let Some(escaped) = char_at(sql, i) {
i += escaped.len_utf8();
}
continue;
}
i += ch.len_utf8();
}
sql.len()
}
/// Skips a PostgreSQL `$tag$...$tag$` dollar-quoted body starting at `pos`
/// (pointing at the first `$`). Returns `None` if `pos` isn't actually the
/// start of a valid dollar-quote tag (e.g. a stray `$` in an expression), in
/// which case the caller should treat `$` as an ordinary character.
fn skip_dollar_quoted(sql: &str, pos: usize) -> Option<usize> {
let tag_end_offset = sql.get(pos + 1..)?.find('$')?;
let tag_end = pos + 1 + tag_end_offset;
let tag = &sql[pos + 1..tag_end];
let valid_tag = tag.is_empty()
|| (tag.chars().next().is_some_and(|ch| ch.is_ascii_alphabetic() || ch == '_')
&& tag.chars().all(|ch| ch.is_ascii_alphanumeric() || ch == '_'));
if !valid_tag {
return None;
}
let delimiter = &sql[pos..=tag_end];
let content_start = tag_end + 1;
sql.get(content_start..)?.find(delimiter).map(|offset| content_start + offset + delimiter.len())
}
/// Finds the byte index just past the first top-level (parenthesis-depth-0)
/// closing paren at or after `from` — the end of the outermost parenthesized
/// group that starts somewhere at or after `from` — treating quoted or
/// dollar-quoted content as opaque so parens or keywords inside a
/// `DEFAULT`/`CHECK` expression can't confuse the count. Returns `None` if no
/// top-level `(` is ever opened, or it's never closed.
pub(crate) fn find_top_level_paren_close(sql: &str, from: usize) -> Option<usize> {
let mut i = from;
let mut depth: usize = 0;
let mut opened = false;
while i < sql.len() {
let ch = char_at(sql, i)?;
match ch {
'\'' | '"' => {
i = skip_quoted(sql, i, ch);
continue;
}
'$' => {
if let Some(end) = skip_dollar_quoted(sql, i) {
i = end;
continue;
}
}
'(' => {
depth += 1;
opened = true;
}
')' => {
depth = depth.saturating_sub(1);
i += ch.len_utf8();
if opened && depth == 0 {
return Some(i);
}
continue;
}
_ => {}
}
i += ch.len_utf8();
}
None
}
/// Finds the byte index of the next top-level (depth-0, outside any quoted
/// or dollar-quoted content) `;` at or after `from`.
fn find_top_level_semicolon(sql: &str, from: usize) -> Option<usize> {
let mut i = from;
let mut depth: usize = 0;
while i < sql.len() {
let ch = char_at(sql, i)?;
match ch {
'\'' | '"' => {
i = skip_quoted(sql, i, ch);
continue;
}
'$' => {
if let Some(end) = skip_dollar_quoted(sql, i) {
i = end;
continue;
}
}
'(' => depth += 1,
')' => depth = depth.saturating_sub(1),
';' if depth == 0 => return Some(i),
_ => {}
}
i += ch.len_utf8();
}
None
}
/// Splits `sql` into consecutive top-level statement ranges, each ending
/// just past its own top-level `;`. The last statement may lack a trailing
/// `;`, in which case its range runs to the end of the string.
pub(crate) fn top_level_statement_ranges(sql: &str) -> Vec<Range<usize>> {
let mut ranges = Vec::new();
let mut start = 0;
while start < sql.len() {
match find_top_level_semicolon(sql, start) {
Some(semi) => {
ranges.push(start..semi + 1);
start = semi + 1;
}
None => {
ranges.push(start..sql.len());
break;
}
}
}
ranges
}
/// If `stmt` (ignoring leading whitespace) begins with `CREATE TABLE
/// "schema"."table"` or `CREATE FOREIGN TABLE "schema"."table"`, returns the
/// unescaped `(schema, table)` pair.
pub(crate) fn parse_create_table_relation(stmt: &str) -> Option<(String, String)> {
let trimmed = stmt.trim_start();
let rest =
trimmed.strip_prefix("CREATE FOREIGN TABLE ").or_else(|| trimmed.strip_prefix("CREATE TABLE "))?.trim_start();
let (schema, rest) = unquote_ident(rest)?;
let (table, _) = unquote_ident(rest.strip_prefix('.')?)?;
Some((schema, table))
}
fn unquote_ident(s: &str) -> Option<(String, &str)> {
if !s.starts_with('"') {
return None;
}
let end = skip_quoted(s, 0, '"');
if end < 2 {
return None;
}
let raw = &s[1..end - 1];
Some((raw.replace("\"\"", "\""), &s[end..]))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn paren_close_skips_nested_parens_and_quotes() {
let ddl =
"\"public\".\"t\" (\n \"a\" text DEFAULT 'x(y)'::text,\n \"b\" text CHECK ((b <> 'z'))\n) SERVER \"s\"";
let close = find_top_level_paren_close(ddl, 0).unwrap();
assert_eq!(
&ddl[..close],
"\"public\".\"t\" (\n \"a\" text DEFAULT 'x(y)'::text,\n \"b\" text CHECK ((b <> 'z'))\n)"
);
}
#[test]
fn paren_close_ignores_literal_server_text_inside_quotes() {
let ddl = "\"public\".\"t\" (\n \"note\" text DEFAULT 'contact SERVER \"admin\"'\n) SERVER \"real_server\"";
let close = find_top_level_paren_close(ddl, 0).unwrap();
assert!(ddl[close..].starts_with(" SERVER \"real_server\""));
}
#[test]
fn top_level_statement_ranges_split_on_depth_zero_semicolons() {
let ddl = "CREATE TABLE \"s\".\"t\" (\n \"a\" text DEFAULT 'x;y'\n);\nALTER TABLE \"s\".\"t\" ADD CONSTRAINT c CHECK (a <> '');";
let ranges = top_level_statement_ranges(ddl);
assert_eq!(ranges.len(), 2);
assert!(ddl[ranges[0].clone()].starts_with("CREATE TABLE"));
assert!(ddl[ranges[0].clone()].ends_with(";"));
assert!(ddl[ranges[1].clone()].trim_start().starts_with("ALTER TABLE"));
}
#[test]
fn parse_create_table_relation_unescapes_doubled_quotes() {
let (schema, table) =
parse_create_table_relation("CREATE TABLE \"pub\"\"lic\".\"t\" (\n \"a\" int\n);").unwrap();
assert_eq!(schema, "pub\"lic");
assert_eq!(table, "t");
}
#[test]
fn parse_create_table_relation_rejects_non_create_table() {
assert!(parse_create_table_relation("ALTER TABLE \"s\".\"t\" ADD CONSTRAINT c CHECK (true);").is_none());
}
}
+1
View File
@@ -3,6 +3,7 @@ pub mod clickhouse_driver;
pub mod cloudberry;
pub mod cloudflare_d1;
pub use cloudflare_d1 as cloudflare_d1_driver;
pub(crate) mod ddl_scan;
pub mod document_result;
pub mod dolt;
pub mod doris;
+120
View File
@@ -1,3 +1,5 @@
use std::collections::HashMap;
use deadpool_postgres::Pool;
use crate::db;
@@ -20,6 +22,24 @@ const TABLE_DISTRIBUTION_SQL: &str = "SELECT c.pclocatortype::text, \
WHERE namespace.nspname = $1 AND relation.relname = $2 \
LIMIT 1";
/// Batched sibling of `TABLE_DISTRIBUTION_SQL`, for appending each
/// partition's own `DISTRIBUTE BY` clause when rendering a whole partition
/// tree instead of just the root relation.
const TABLE_DISTRIBUTION_FOR_RELATIONS_SQL: &str =
"SELECT namespace.nspname, relation.relname, c.pclocatortype::text, \
COALESCE(( \
SELECT string_agg(a.attname, E'\\n' ORDER BY distributed_column.ordinality) \
FROM unnest(c.discolnums::smallint[]) WITH ORDINALITY \
AS distributed_column(attnum, ordinality) \
JOIN pg_catalog.pg_attribute a \
ON a.attrelid = c.pcrelid AND a.attnum = distributed_column.attnum \
WHERE NOT a.attisdropped \
), '')::text \
FROM unnest($1::text[], $2::text[]) AS rel(rel_schema, rel_table) \
JOIN pg_catalog.pg_namespace namespace ON namespace.nspname = rel.rel_schema \
JOIN pg_catalog.pg_class relation ON relation.relnamespace = namespace.oid AND relation.relname = rel.rel_table \
JOIN pg_catalog.pgxc_class c ON c.pcrelid = relation.oid";
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DistributionPolicy {
Hash(Vec<String>),
@@ -52,6 +72,43 @@ pub async fn table_distribution(pool: &Pool, schema: &str, table: &str) -> Resul
parse_distribution_policy(&locator_type, columns)
}
/// Batched sibling of `table_distribution`, for a whole partition tree.
/// Relations without a distribution policy (or that don't exist) are simply
/// absent from the result map rather than erroring the whole batch.
pub async fn table_distribution_for_relations(
pool: &Pool,
relations: &[(String, String)],
) -> Result<HashMap<(String, String), DistributionPolicy>, String> {
if relations.is_empty() {
return Ok(HashMap::new());
}
let client = db::postgres::checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let schemas: Vec<&str> = relations.iter().map(|(schema, _)| schema.as_str()).collect();
let tables: Vec<&str> = relations.iter().map(|(_, table)| table.as_str()).collect();
let rows = client
.query(TABLE_DISTRIBUTION_FOR_RELATIONS_SQL, &[&schemas, &tables])
.await
.map_err(|error| error.to_string())?;
let mut result = HashMap::new();
for row in &rows {
let schema = row.try_get::<_, String>(0).map_err(|error| error.to_string())?;
let table = row.try_get::<_, String>(1).map_err(|error| error.to_string())?;
let locator_type = row.try_get::<_, String>(2).map_err(|error| error.to_string())?;
let columns = row
.try_get::<_, String>(3)
.map_err(|error| error.to_string())?
.lines()
.map(str::trim)
.filter(|column| !column.is_empty())
.map(str::to_string)
.collect::<Vec<_>>();
if let Some(distribution) = parse_distribution_policy(&locator_type, columns)? {
result.insert((schema, table), distribution);
}
}
Ok(result)
}
pub fn append_distribution_clause(ddl: &str, distribution: &DistributionPolicy) -> Result<String, String> {
let clause = render_distribution_clause(distribution)?;
let insertion = ddl
@@ -66,6 +123,41 @@ pub fn append_distribution_clause(ddl: &str, distribution: &DistributionPolicy)
Ok(output)
}
/// Tree-aware sibling of `append_distribution_clause`: `ddl` may contain
/// several relations' worth of statements concatenated together (root plus
/// every partition), so each `CREATE [FOREIGN] TABLE "schema"."table"`
/// statement gets its own `DISTRIBUTE BY` clause inserted before its own
/// terminator, looked up by that statement's own `(schema, table)` — instead
/// of inserting a single clause at the first `;` in the whole string, which
/// would only ever land on the root relation's statement.
pub fn append_distribution_clauses(
ddl: &str,
distributions: &HashMap<(String, String), DistributionPolicy>,
) -> Result<String, String> {
if distributions.is_empty() {
return Ok(ddl.to_string());
}
let mut output = String::with_capacity(ddl.len() + distributions.len() * 32);
let mut cursor = 0usize;
for range in db::ddl_scan::top_level_statement_ranges(ddl) {
let statement = &ddl[range.clone()];
let Some(relation) = db::ddl_scan::parse_create_table_relation(statement) else {
continue;
};
let Some(distribution) = distributions.get(&relation) else {
continue;
};
let clause = render_distribution_clause(distribution)?;
let insertion = if statement.ends_with(';') { range.end - 1 } else { range.end };
output.push_str(&ddl[cursor..insertion]);
output.push('\n');
output.push_str(&clause);
cursor = insertion;
}
output.push_str(&ddl[cursor..]);
Ok(output)
}
fn parse_distribution_policy(locator_type: &str, columns: Vec<String>) -> Result<Option<DistributionPolicy>, String> {
let column_distribution = |kind| {
if columns.is_empty() {
@@ -148,4 +240,32 @@ mod tests {
assert!(TABLE_DISTRIBUTION_SQL.contains("WITH ORDINALITY"));
assert!(TABLE_DISTRIBUTION_SQL.contains("NOT a.attisdropped"));
}
#[test]
fn batched_distribution_query_matches_opentenbase_catalog_contract() {
assert!(TABLE_DISTRIBUTION_FOR_RELATIONS_SQL.contains("unnest($1::text[], $2::text[])"));
assert!(TABLE_DISTRIBUTION_FOR_RELATIONS_SQL.contains("pg_catalog.pgxc_class"));
assert!(TABLE_DISTRIBUTION_FOR_RELATIONS_SQL.contains("WITH ORDINALITY"));
}
#[test]
fn appends_each_partition_its_own_distribution_clause_in_a_multi_relation_tree() {
let ddl = "CREATE TABLE \"public\".\"events\" (\n \"id\" integer\n) PARTITION BY RANGE (id);\n\nCREATE TABLE \"public\".\"events_2026\" PARTITION OF \"public\".\"events\" FOR VALUES FROM (1) TO (100);\n\nCREATE TABLE \"public\".\"events_2027\" PARTITION OF \"public\".\"events\" FOR VALUES FROM (100) TO (200);\n";
let mut distributions = HashMap::new();
distributions
.insert(("public".to_string(), "events".to_string()), DistributionPolicy::Hash(vec!["id".to_string()]));
distributions.insert(("public".to_string(), "events_2027".to_string()), DistributionPolicy::Replication);
// "events_2026" deliberately has no entry — it must be left untouched.
let rendered = append_distribution_clauses(ddl, &distributions).unwrap();
assert!(rendered.contains("PARTITION BY RANGE (id)\nDISTRIBUTE BY HASH (\"id\");"));
assert!(rendered.contains(
"CREATE TABLE \"public\".\"events_2027\" PARTITION OF \"public\".\"events\" FOR VALUES FROM (100) TO (200)\nDISTRIBUTE BY REPLICATION;"
));
assert!(rendered.contains(
"CREATE TABLE \"public\".\"events_2026\" PARTITION OF \"public\".\"events\" FOR VALUES FROM (1) TO (100);"
));
assert_eq!(rendered.matches("DISTRIBUTE BY").count(), 2);
}
}
+96 -74
View File
@@ -1018,6 +1018,37 @@ fn pg_error_to_string(err: tokio_postgres::Error) -> String {
err.as_db_error().map(ToString::to_string).unwrap_or_else(|| err.to_string())
}
/// Tries each SQL tier in `tiers` in order (most-capable first), via `run`,
/// returning the first tier that succeeds. Every driver-compat query in this
/// module (a "does this server have the newer catalog column" primary/compat
/// split, occasionally with a further information_schema fallback) used to
/// hand-roll this same try/log/combine-errors shape once per query; this is
/// the shared version.
///
/// If every tier fails, all of their errors are logged together at debug
/// level (so a fallback firing in production is diagnosable) and the last
/// tier's error is returned to the caller, since it's usually the most
/// specific one for whatever the connected server actually is.
async fn query_with_compat_fallback<T, F, Fut>(
log_context: &str,
tiers: &[&'static str],
mut run: F,
) -> Result<T, String>
where
F: FnMut(&'static str) -> Fut,
Fut: std::future::Future<Output = Result<T, tokio_postgres::Error>>,
{
let mut errors: Vec<String> = Vec::new();
for sql in tiers {
match run(sql).await {
Ok(value) => return Ok(value),
Err(error) => errors.push(pg_error_to_string(error)),
}
}
log::debug!("[postgres][{log_context}:compat-failed] {}", errors.join("; "));
Err(errors.into_iter().next_back().unwrap_or_else(|| format!("[postgres][{log_context}] no SQL tiers configured")))
}
fn pg_db_error_to_string(err: &tokio_postgres::error::DbError) -> String {
format!("{err} (SQLSTATE {})", err.code().code())
}
@@ -3243,23 +3274,11 @@ pub async fn get_columns_for_relations(
) -> Result<HashMap<i64, Vec<ColumnInfo>>, String> {
let oids: Vec<i64> = relations.iter().map(|(oid, _, _)| *oid).collect();
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let [primary_sql, compat_sql] = postgres_columns_for_relations_query_tiers();
match get_columns_for_relations_with_sql(&client, primary_sql, &oids).await {
Ok(columns) => Ok(columns),
Err(primary_error) => match get_columns_for_relations_with_sql(&client, compat_sql, &oids).await {
Ok(columns) => Ok(columns),
Err(fallback_error) => {
let primary_message = pg_error_to_string(primary_error);
let fallback_message = pg_error_to_string(fallback_error);
log::debug!(
"[postgres][get_columns_for_relations:compat-failed] primary_error={} fallback_error={}",
primary_message,
fallback_message
);
Err(fallback_message)
}
},
}
let tiers = postgres_columns_for_relations_query_tiers();
query_with_compat_fallback("get_columns_for_relations", &tiers, |sql| {
get_columns_for_relations_with_sql(&client, sql, &oids)
})
.await
}
async fn get_columns_for_relations_with_sql(
@@ -3280,6 +3299,15 @@ fn postgres_columns_for_relations_query_tiers() -> [&'static str; 2] {
[postgres_columns_for_relations_sql(), postgres_columns_for_relations_compat_sql()]
}
// Sibling of `POSTGRES_COLUMNS_SQL`/`POSTGRES_COLUMNS_COMPAT_SQL` below (~line
// 4880): same column list and detection logic (identity/serial inference,
// numeric precision/scale, enum values), batched by oid instead of a single
// (schema, table) pair. Kept as a separate literal rather than sharing a
// fragment — `pg_class` needs the `c` alias here for the oid filter, which
// pushes `information_schema.columns` to `ic` instead of the single-relation
// version's `c`, and every column's position in the SELECT list is relied on
// positionally by `column_info_from_row_offset`. A change to one almost
// certainly needs the same change in the other.
fn postgres_columns_for_relations_sql() -> &'static str {
"SELECT c.oid::bigint AS relid, a.attname AS column_name, \
format_type(a.atttypid, a.atttypmod) AS full_type, \
@@ -3352,6 +3380,8 @@ fn postgres_columns_for_relations_sql() -> &'static str {
ORDER BY c.oid, a.attnum"
}
// Compat-tier sibling of `POSTGRES_COLUMNS_COMPAT_SQL` (~line 4938) — see the
// note on `postgres_columns_for_relations_sql` above.
fn postgres_columns_for_relations_compat_sql() -> &'static str {
"SELECT c.oid::bigint AS relid, a.attname AS column_name, \
format_type(a.atttypid, a.atttypmod) AS full_type, \
@@ -3438,23 +3468,11 @@ pub async fn list_indexes_for_relations(
) -> Result<HashMap<i64, Vec<IndexInfo>>, String> {
let oids: Vec<i64> = relations.iter().map(|(oid, _, _)| *oid).collect();
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let [primary_sql, compat_sql] = postgres_indexes_for_relations_query_tiers();
match list_indexes_for_relations_with_sql(&client, primary_sql, &oids).await {
Ok(indexes) => Ok(indexes),
Err(primary_error) => match list_indexes_for_relations_with_sql(&client, compat_sql, &oids).await {
Ok(indexes) => Ok(indexes),
Err(fallback_error) => {
let primary_message = pg_error_to_string(primary_error);
let fallback_message = pg_error_to_string(fallback_error);
log::debug!(
"[postgres][list_indexes_for_relations:compat-failed] primary_error={} fallback_error={}",
primary_message,
fallback_message
);
Err(fallback_message)
}
},
}
let tiers = postgres_indexes_for_relations_query_tiers();
query_with_compat_fallback("list_indexes_for_relations", &tiers, |sql| {
list_indexes_for_relations_with_sql(&client, sql, &oids)
})
.await
}
async fn list_indexes_for_relations_with_sql(
@@ -3493,6 +3511,11 @@ fn postgres_indexes_for_relations_query_tiers() -> [&'static str; 2] {
[postgres_indexes_for_relations_sql(), postgres_indexes_for_relations_compat_sql()]
}
// Sibling of `POSTGRES_INDEXES_SQL`/`POSTGRES_INDEXES_COMPAT_SQL` (~line
// 6042): same index-detection logic, batched by oid instead of a single
// (schema, table) pair. Not merged into a shared fragment for the same
// reason as the columns queries above — an alias would need renaming to
// line up, and result columns are read positionally.
fn postgres_indexes_for_relations_sql() -> &'static str {
"SELECT t.oid::bigint AS relid, i.relname AS index_name, \
array_agg(COALESCE(a.attname, pg_get_indexdef(ix.indexrelid, k.n::int, true)) ORDER BY k.n) AS columns, \
@@ -3515,6 +3538,8 @@ fn postgres_indexes_for_relations_sql() -> &'static str {
ORDER BY t.oid, i.relname"
}
// Compat-tier sibling of `POSTGRES_INDEXES_COMPAT_SQL` (~line 6063) — see the
// note on `postgres_indexes_for_relations_sql` above.
fn postgres_indexes_for_relations_compat_sql() -> &'static str {
"SELECT t.oid::bigint AS relid, i.relname AS index_name, \
ARRAY( \
@@ -5101,6 +5126,9 @@ pub async fn list_schema_infos_with_system(pool: &Pool, show_system_schemas: boo
.collect())
}
// Sibling of `postgres_columns_for_relations_sql`/`_compat_sql` above (~line
// 3086), for a single (schema, table) instead of a batch of oids — see the
// note there about why these aren't merged into a shared fragment.
const POSTGRES_COLUMNS_SQL: &str = "SELECT a.attname AS column_name, \
format_type(a.atttypid, a.atttypmod) AS full_type, \
COALESCE(c.is_nullable = 'YES', NOT a.attnotnull) AS is_nullable, \
@@ -5169,6 +5197,8 @@ const POSTGRES_COLUMNS_SQL: &str = "SELECT a.attname AS column_name, \
AND a.attnum > 0 AND NOT a.attisdropped \
ORDER BY a.attnum";
// Compat-tier sibling of `postgres_columns_for_relations_compat_sql` (~line
// 3148) — see the note on `POSTGRES_COLUMNS_SQL` above.
const POSTGRES_COLUMNS_COMPAT_SQL: &str = "SELECT a.attname AS column_name, \
format_type(a.atttypid, a.atttypmod) AS full_type, \
COALESCE(c.is_nullable = 'YES', NOT a.attnotnull) AS is_nullable, \
@@ -5348,29 +5378,8 @@ async fn get_columns_with_sql(
pub async fn get_columns(pool: &Pool, schema: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
let schema = if schema.is_empty() { "public" } else { schema };
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
match get_columns_with_sql(&client, POSTGRES_COLUMNS_SQL, schema, table).await {
Ok(columns) => Ok(columns),
Err(primary_error) => match get_columns_with_sql(&client, POSTGRES_COLUMNS_COMPAT_SQL, schema, table).await {
Ok(columns) => Ok(columns),
Err(fallback_error) => {
let primary_message = pg_error_to_string(primary_error);
let fallback_message = pg_error_to_string(fallback_error);
match get_columns_with_sql(&client, POSTGRES_COLUMNS_INFORMATION_SCHEMA_SQL, schema, table).await {
Ok(columns) => Ok(columns),
Err(information_schema_error) => {
let information_schema_message = pg_error_to_string(information_schema_error);
log::debug!(
"[postgres][get_columns:compat-failed] primary_error={} fallback_error={} information_schema_error={}",
primary_message,
fallback_message,
information_schema_message
);
Err(information_schema_message)
}
}
}
},
}
let tiers = [POSTGRES_COLUMNS_SQL, POSTGRES_COLUMNS_COMPAT_SQL, POSTGRES_COLUMNS_INFORMATION_SCHEMA_SQL];
query_with_compat_fallback("get_columns", &tiers, |sql| get_columns_with_sql(&client, sql, schema, table)).await
}
fn pg_quote_literal(value: &str) -> String {
@@ -6372,6 +6381,8 @@ async fn execute_query_with_max_rows_inner(
}
}
// Sibling of `postgres_indexes_for_relations_sql` (~line 3288), for a single
// (schema, table) instead of a batch of oids — see the note there.
const POSTGRES_INDEXES_SQL: &str = "SELECT i.relname AS index_name, \
array_agg(COALESCE(a.attname, pg_get_indexdef(ix.indexrelid, k.n::int, true)) ORDER BY k.n) AS columns, \
ix.indisunique AS is_unique, \
@@ -6393,6 +6404,8 @@ const POSTGRES_INDEXES_SQL: &str = "SELECT i.relname AS index_name, \
GROUP BY i.relname, i.oid, ix.indisunique, ix.indisprimary, ix.indpred, ix.indrelid, am.amname, ix.indnkeyatts, ix.indkey \
ORDER BY i.relname";
// Compat-tier sibling of `postgres_indexes_for_relations_compat_sql` (~line
// 3312) — see the note on `POSTGRES_INDEXES_SQL` above.
const POSTGRES_INDEXES_COMPAT_SQL: &str = "SELECT i.relname AS index_name, \
ARRAY( \
SELECT COALESCE(a.attname, pg_get_indexdef(ix.indexrelid, pos.n, true)) \
@@ -6465,7 +6478,7 @@ const POSTGRES_COLUMN_ACL_PRIVILEGES_SQL: &str =
WHERE n.nspname = $1 AND c.relname = $2 AND c.relkind IN ('r', 'p') \
ORDER BY 5, 1, 2, 3, 4";
fn postgres_owner_object_type(relkind: &str) -> &str {
pub(crate) fn postgres_owner_object_type(relkind: &str) -> &str {
match relkind {
"r" => "TABLE",
"v" => "VIEW",
@@ -6478,6 +6491,29 @@ fn postgres_owner_object_type(relkind: &str) -> &str {
}
}
/// Looks up a relation's own `relkind` (e.g. `'r'`, `'v'`, `'S'`), regardless
/// of whether it's a kind this driver otherwise treats as a table. Used to
/// give a specific diagnostic ("it's a view, not a table") instead of a bare
/// "not found" when a `(schema, table)` request turns out not to be a
/// table/partition/foreign table.
pub(crate) async fn postgres_relation_relkind(
pool: &Pool,
schema: &str,
table: &str,
) -> Result<Option<String>, String> {
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let row = client
.query_opt(
"SELECT c.relkind::text FROM pg_catalog.pg_class c \
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \
WHERE n.nspname = $1 AND c.relname = $2 LIMIT 1",
&[&schema, &table],
)
.await
.map_err(pg_error_to_string)?;
row.map(|row| row.try_get::<_, String>(0)).transpose().map_err(pg_error_to_string)
}
async fn list_indexes_with_sql(
client: &deadpool_postgres::Client,
sql: &str,
@@ -6516,22 +6552,8 @@ async fn list_indexes_with_sql(
pub async fn list_indexes(pool: &Pool, schema: &str, table: &str) -> Result<Vec<IndexInfo>, String> {
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
match list_indexes_with_sql(&client, POSTGRES_INDEXES_SQL, schema, table).await {
Ok(indexes) => Ok(indexes),
Err(primary_error) => match list_indexes_with_sql(&client, POSTGRES_INDEXES_COMPAT_SQL, schema, table).await {
Ok(indexes) => Ok(indexes),
Err(fallback_error) => {
let primary_message = pg_error_to_string(primary_error);
let fallback_message = pg_error_to_string(fallback_error);
log::debug!(
"[postgres][list_indexes:compat-failed] primary_error={} fallback_error={}",
primary_message,
fallback_message
);
Err(fallback_message)
}
},
}
let tiers = [POSTGRES_INDEXES_SQL, POSTGRES_INDEXES_COMPAT_SQL];
query_with_compat_fallback("list_indexes", &tiers, |sql| list_indexes_with_sql(&client, sql, schema, table)).await
}
/// Names of same-table indexes whose `pg_index.indisvalid` is `false`.
+131 -9
View File
@@ -9812,9 +9812,17 @@ pub async fn pg_ddl_with_partitions(
let Some(root) =
tree.iter().find(|node| !node.parent_oid.is_some_and(|parent_oid| tree_oids.contains(&parent_oid)))
else {
return Err(format!(
"relation \"{schema}\".\"{table}\" was not found or is not a table/partition/foreign table"
));
// The tree query only matches relkind IN ('r','p','f'), so an empty
// tree could mean the relation doesn't exist at all, or that it
// exists but is a view/sequence/other non-table object. Look up its
// relkind (if any) to tell those two cases apart in the error.
return Err(match db::postgres::postgres_relation_relkind(pool, schema, table).await {
Ok(Some(relkind)) => format!(
"relation \"{schema}\".\"{table}\" is a {}, not a table/partition/foreign table",
db::postgres::postgres_owner_object_type(&relkind)
),
_ => format!("relation \"{schema}\".\"{table}\" was not found or is not a table/partition/foreign table"),
});
};
let oids: Vec<i64> = tree.iter().map(|node| node.oid).collect();
@@ -9841,6 +9849,18 @@ pub async fn pg_ddl_with_partitions(
db::postgres::get_table_partition_local_objects_for_relations(pool, &oids),
)?;
// `list_foreign_keys_for_relations` is keyed by (schema, table) since its
// query is information_schema-based, unlike every other *_by_oid map
// here. Remap it once so the render loop below can look fkeys up by oid
// like everything else, instead of cloning a fresh (schema, table) key
// on every node it renders.
let relation_oid_by_pair: HashMap<(String, String), i64> =
relations.into_iter().map(|(oid, schema, table)| ((schema, table), oid)).collect();
let fkeys_by_oid: HashMap<i64, Vec<db::ForeignKeyInfo>> = fkeys_by_relation
.into_iter()
.filter_map(|(key, fkeys)| relation_oid_by_pair.get(&key).map(|oid| (*oid, fkeys)))
.collect();
// Group children by parent oid, each group ordered by relname to match
// `list_table_partitions`' `ORDER BY c.relname` (today's traversal order).
let mut children_by_parent: HashMap<i64, Vec<&db::postgres::PostgresPartitionTreeNode>> = HashMap::new();
@@ -9859,7 +9879,7 @@ pub async fn pg_ddl_with_partitions(
&children_by_parent,
&columns_by_oid,
&indexes_by_oid,
&fkeys_by_relation,
&fkeys_by_oid,
&comments_by_oid,
&triggers_by_oid,
&checks_by_oid,
@@ -9880,7 +9900,7 @@ fn render_postgres_partition_tree_node(
children_by_parent: &HashMap<i64, Vec<&db::postgres::PostgresPartitionTreeNode>>,
columns_by_oid: &HashMap<i64, Vec<db::ColumnInfo>>,
indexes_by_oid: &HashMap<i64, Vec<db::IndexInfo>>,
fkeys_by_relation: &HashMap<(String, String), Vec<db::ForeignKeyInfo>>,
fkeys_by_oid: &HashMap<i64, Vec<db::ForeignKeyInfo>>,
comments_by_oid: &HashMap<i64, Option<String>>,
triggers_by_oid: &HashMap<i64, Vec<String>>,
checks_by_oid: &HashMap<i64, Vec<(String, String)>>,
@@ -9903,8 +9923,8 @@ fn render_postgres_partition_tree_node(
let columns = columns_by_oid.get(&node.oid).unwrap_or(&empty_columns);
let indexes = indexes_by_oid.get(&node.oid).unwrap_or(&empty_indexes);
let fkeys = fkeys_by_relation.get(&(node.schema.clone(), node.table.clone())).unwrap_or(&empty_fkeys);
let comment = comments_by_oid.get(&node.oid).cloned().flatten();
let fkeys = fkeys_by_oid.get(&node.oid).unwrap_or(&empty_fkeys);
let comment = comments_by_oid.get(&node.oid).and_then(|comment| comment.as_deref());
let triggers = triggers_by_oid.get(&node.oid).unwrap_or(&empty_triggers);
let checks = checks_by_oid.get(&node.oid).unwrap_or(&empty_checks);
let local_objects = local_objects_by_oid.get(&node.oid).unwrap_or(&empty_local_objects);
@@ -9920,7 +9940,7 @@ fn render_postgres_partition_tree_node(
indexes,
fkeys,
checks,
comment.as_deref(),
comment,
&node.partition_info,
local_objects,
),
@@ -10195,13 +10215,82 @@ fn append_opengauss_trigger_definitions(mut ddl: String, trigger_definitions: &[
ddl
}
/// Cloudberry's native `pg_get_tabledef` (a community-maintained PL/pgSQL
/// function most Cloudberry/Greenplum installs have, not a built-in) renders
/// exactly one relation — for a partitioned table it emits that table's own
/// `CREATE TABLE ... PARTITION BY ...`, never its partitions' own `CREATE
/// TABLE ... PARTITION OF ...` statements, regardless of whether the table
/// was created with classic Greenplum or PostgreSQL-style declarative
/// partition syntax. When `include_partitions` is requested, fetch the same
/// partition tree the plain-Postgres path uses and append each descendant's
/// own native DDL, so Cloudberry's more accurate native rendering (storage
/// options, distribution policy, external-table clauses) is still used per
/// relation instead of falling back to the generic renderer for the whole
/// tree.
async fn cloudberry_native_tree_ddl(
pool: &deadpool_postgres::Pool,
schema: &str,
table: &str,
include_partitions: bool,
) -> Result<String, String> {
let root_ddl = db::cloudberry::table_ddl(pool, schema, table).await?;
if !include_partitions {
return Ok(root_ddl);
}
let tree = db::postgres::fetch_postgres_partition_tree(pool, schema, table).await?;
if tree.len() <= 1 {
return Ok(root_ddl);
}
let tree_oids: HashSet<i64> = tree.iter().map(|node| node.oid).collect();
let Some(root) =
tree.iter().find(|node| !node.parent_oid.is_some_and(|parent_oid| tree_oids.contains(&parent_oid)))
else {
return Ok(root_ddl);
};
let mut children_by_parent: HashMap<i64, Vec<&db::postgres::PostgresPartitionTreeNode>> = HashMap::new();
for node in &tree {
if let Some(parent_oid) = node.parent_oid {
children_by_parent.entry(parent_oid).or_default().push(node);
}
}
for children in children_by_parent.values_mut() {
children.sort_by(|a, b| a.table.cmp(&b.table));
}
let mut descendants = Vec::new();
let mut visited: HashSet<i64> = HashSet::new();
let mut stack: Vec<&db::postgres::PostgresPartitionTreeNode> = vec![root];
while let Some(node) = stack.pop() {
if !visited.insert(node.oid) {
continue;
}
if node.oid != root.oid {
descendants.push(node);
}
if let Some(children) = children_by_parent.get(&node.oid) {
for child in children.iter().rev() {
stack.push(child);
}
}
}
let mut ddl = root_ddl;
for node in descendants {
let child_ddl = db::cloudberry::table_ddl(pool, &node.schema, &node.table).await?;
ddl.push('\n');
ddl.push_str(&child_ddl);
}
Ok(ddl)
}
pub async fn cloudberry_ddl(
pool: &deadpool_postgres::Pool,
schema: &str,
table: &str,
include_partitions: bool,
) -> Result<String, String> {
match db::cloudberry::table_ddl(pool, schema, table).await {
match cloudberry_native_tree_ddl(pool, schema, table, include_partitions).await {
Ok(ddl) => Ok(ddl),
Err(native_error) => {
let base_ddl = pg_ddl_for_options(pool, schema, table, include_partitions).await.map_err(|fallback_error| {
@@ -10228,6 +10317,39 @@ pub async fn opentenbase_ddl(
include_partitions: bool,
) -> Result<String, String> {
let ddl = pg_ddl_for_options(pool, schema, table, include_partitions).await?;
if include_partitions {
// The rendered ddl may cover the whole partition tree (root plus
// every partition), each as its own `CREATE [FOREIGN] TABLE`
// statement, so distribution policies need to be looked up and
// applied per relation rather than once for the root.
let relations: Vec<(String, String)> = db::ddl_scan::top_level_statement_ranges(&ddl)
.into_iter()
.filter_map(|range| db::ddl_scan::parse_create_table_relation(&ddl[range]))
.collect();
return match db::opentenbase::table_distribution_for_relations(pool, &relations).await {
Ok(distributions) => match db::opentenbase::append_distribution_clauses(&ddl, &distributions) {
Ok(ddl) => Ok(ddl),
Err(error) => {
log::warn!(
"[schema][opentenbase:table-ddl-distribution-render-fallback] schema={} table={} error={}",
schema,
table,
error
);
Ok(ddl)
}
},
Err(error) => {
log::warn!(
"[schema][opentenbase:table-ddl-distribution-query-fallback] schema={} table={} error={}",
schema,
table,
error
);
Ok(ddl)
}
};
}
match db::opentenbase::table_distribution(pool, schema, table).await {
Ok(Some(distribution)) => match db::opentenbase::append_distribution_clause(&ddl, &distribution) {
Ok(ddl) => Ok(ddl),
@@ -10,164 +10,17 @@
//! number of partitions.
//! Needs local Docker; skips cleanly when unavailable.
mod support;
use std::process::Command;
use dbx_core::connection::AppState;
use dbx_core::database_export::{export_database_sql_core, DatabaseExportRequest};
use dbx_core::models::connection::{ConnectionConfig, DatabaseType};
use dbx_core::storage::Storage;
struct DockerPostgres {
name: String,
port: u16,
remove_on_drop: bool,
}
impl Drop for DockerPostgres {
fn drop(&mut self) {
if self.remove_on_drop {
let _ = Command::new("docker").args(["rm", "-f", &self.name]).status();
}
}
}
fn docker_ready() -> bool {
Command::new("docker")
.args(["version", "--format", "{{.Server.Version}}"])
.output()
.map(|output| output.status.success())
.unwrap_or(false)
}
fn start_docker_postgres(name_prefix: &str) -> Option<DockerPostgres> {
if let Ok(name) = std::env::var("DBX_TEST_POSTGRES_CONTAINER") {
let port = std::env::var("DBX_TEST_POSTGRES_PORT")
.expect("DBX_TEST_POSTGRES_PORT must accompany DBX_TEST_POSTGRES_CONTAINER")
.parse()
.expect("DBX_TEST_POSTGRES_PORT must be a valid port");
return Some(DockerPostgres { name, port, remove_on_drop: false });
}
if !docker_ready() {
eprintln!("skipping docker-backed partition ddl test because Docker is unavailable");
return None;
}
let port = portpicker::pick_unused_port().expect("pick unused postgres port");
let container =
DockerPostgres { name: format!("{name_prefix}-{}", uuid::Uuid::new_v4()), port, remove_on_drop: true };
let status = Command::new("docker")
.args([
"run",
"-d",
"--rm",
"--name",
&container.name,
"-e",
"POSTGRES_PASSWORD=postgres",
"-e",
"POSTGRES_USER=postgres",
"-e",
"POSTGRES_DB=postgres",
"-p",
&format!("{port}:5432"),
"postgres:16-alpine",
])
.status()
.expect("start docker postgres");
assert!(status.success(), "docker run postgres container should succeed");
let mut consecutive_ok = 0;
for _ in 0..120 {
let ready = Command::new("docker")
.args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-c", "SELECT 1"])
.output()
.map(|output| output.status.success())
.unwrap_or(false);
consecutive_ok = if ready { consecutive_ok + 1 } else { 0 };
if consecutive_ok >= 2 {
return Some(container);
}
std::thread::sleep(std::time::Duration::from_millis(500));
}
panic!("postgres container did not become ready");
}
fn psql(container: &DockerPostgres, sql: &str) {
let output = Command::new("docker")
.args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-c", sql])
.output()
.expect("run psql");
assert!(output.status.success(), "psql failed: {}", String::from_utf8_lossy(&output.stderr));
}
fn psql_allow_failure(container: &DockerPostgres, sql: &str) -> bool {
Command::new("docker")
.args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-c", sql])
.output()
.map(|output| output.status.success())
.unwrap_or(false)
}
fn postgres_test_config(id: &str, port: u16) -> ConnectionConfig {
ConnectionConfig {
docs_notes_path: None,
id: id.to_string(),
name: id.to_string(),
note: String::new(),
db_type: DatabaseType::Postgres,
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "127.0.0.1".to_string(),
port,
username: "postgres".to_string(),
password: "postgres".to_string(),
database: Some("postgres".to_string()),
default_schema: None,
visible_databases: None,
visible_schemas: None,
attached_databases: Vec::new(),
init_script: None,
color: None,
transport_layers: Vec::new(),
connect_timeout_secs: 5,
query_timeout_secs: 30,
idle_timeout_secs: 60,
keepalive_interval_secs: 0,
ssl: false,
ca_cert_path: String::new(),
client_cert_path: String::new(),
client_key_path: String::new(),
sysdba: false,
oracle_connection_type: None,
connection_string: None,
redis_connection_mode: None,
redis_sentinel_master: String::new(),
redis_sentinel_nodes: String::new(),
redis_sentinel_username: String::new(),
redis_sentinel_password: String::new(),
redis_sentinel_tls: false,
redis_cluster_nodes: String::new(),
redis_key_separator: dbx_core::models::connection::default_redis_key_separator(),
redis_scan_page_size: None,
redis_database_aliases: Default::default(),
etcd_endpoints: String::new(),
gbase_server: String::new(),
informix_server: String::new(),
external_config: None,
jdbc_driver_class: None,
jdbc_driver_paths: Vec::new(),
one_time: false,
save_password: true,
read_only: false,
is_production: false,
production_databases: vec![],
show_system_schemas: false,
database_info: None,
}
}
use support::{
postgres_test_config, psql, psql_allow_failure, start_docker_postgres, start_docker_postgres_with_server_args,
DockerPostgres,
};
/// Full database export of a partitioned schema must not duplicate any
/// relation's `CREATE TABLE`/`CREATE FOREIGN TABLE`, must faithfully replay
@@ -433,3 +286,156 @@ async fn partition_tree_ddl_query_count_does_not_scale_with_partition_count() {
"query count should not scale with partition count: small={small_count} large={large_count}"
);
}
/// A tree-DDL request for a relation that exists but isn't a
/// table/partition/foreign table (e.g. a view) must say so specifically,
/// not fall back to a generic "not found" message that also covers a
/// relation that doesn't exist at all.
#[tokio::test]
async fn display_ddl_error_distinguishes_wrong_relkind_from_missing_relation() {
let Some(container) = start_docker_postgres("dbx-partition-ddl-relkind-error") else {
return;
};
psql(&container, "CREATE VIEW v AS SELECT 1 AS id");
let dir = std::env::temp_dir().join(format!("dbx-partition-relkind-error-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let connection_id = "partition-relkind-error-conn";
state.configs.write().await.insert(connection_id.to_string(), postgres_test_config(connection_id, container.port));
let view_error =
dbx_core::schema::get_table_display_ddl_core(&state, connection_id, "postgres", "public", "v", None)
.await
.expect_err("requesting tree DDL for a view must fail");
assert!(view_error.contains("is a VIEW"), "expected a relkind-specific error, got: {view_error}");
let missing_error = dbx_core::schema::get_table_display_ddl_core(
&state,
connection_id,
"postgres",
"public",
"does_not_exist",
None,
)
.await
.expect_err("requesting tree DDL for a missing relation must fail");
assert!(missing_error.contains("was not found"), "expected a not-found error, got: {missing_error}");
assert!(!missing_error.contains("is a"), "not-found error should not claim a relkind: {missing_error}");
}
/// Cloudberry/Greenplum's `pg_get_tabledef` (a community PL/pgSQL function
/// most installs have, not a catalog builtin) renders exactly the one
/// relation it's asked for — never its partitions' own `CREATE TABLE ...
/// PARTITION OF ...` statements. `cloudberry_ddl(..., include_partitions:
/// true)` must still return every partition's DDL by walking the same
/// partition tree the plain-Postgres path uses, instead of silently
/// returning just the root's DDL whenever the native call succeeds. This
/// installs a minimal stand-in `pg_get_tabledef` (mirroring the real one's
/// single-relation behavior) rather than the full upstream script, since
/// only that single-relation behavior is what the fix under test reacts to.
#[tokio::test]
async fn cloudberry_native_ddl_still_includes_every_partition() {
let Some(container) = start_docker_postgres("dbx-cloudberry-partition-ddl") else {
return;
};
psql(
&container,
"CREATE OR REPLACE FUNCTION pg_get_tabledef(in_schema text, in_table text, _verbose boolean) \
RETURNS text AS $$ SELECT format('CREATE TABLE \"%s\".\"%s\" (id integer) /* stub-native-ddl */;', in_schema, in_table) $$ \
LANGUAGE sql",
);
psql(&container, "CREATE TABLE t (id integer NOT NULL) PARTITION BY RANGE (id)");
psql(&container, "CREATE TABLE t_p1 PARTITION OF t FOR VALUES FROM (0) TO (100)");
psql(&container, "CREATE TABLE t_p2 PARTITION OF t FOR VALUES FROM (100) TO (200)");
let url = format!("postgres://postgres:postgres@127.0.0.1:{}/postgres", container.port);
let pool = dbx_core::db::postgres::connect(&url, std::time::Duration::from_secs(5)).await.expect("connect");
let ddl =
dbx_core::schema::cloudberry_ddl(&pool, "public", "t", true).await.expect("cloudberry ddl with partitions");
assert_eq!(
ddl.matches("stub-native-ddl").count(),
3,
"expected the root plus both partitions to each get their own native DDL: {ddl}"
);
assert!(ddl.contains("CREATE TABLE \"public\".\"t\""));
assert!(ddl.contains("CREATE TABLE \"public\".\"t_p1\""));
assert!(ddl.contains("CREATE TABLE \"public\".\"t_p2\""));
let single_ddl =
dbx_core::schema::cloudberry_ddl(&pool, "public", "t", false).await.expect("cloudberry ddl without partitions");
assert_eq!(single_ddl.matches("stub-native-ddl").count(), 1, "expected only the root's own DDL: {single_ddl}");
}
/// `opentenbase_ddl(..., include_partitions: true)` must query and append
/// each partition's own `DISTRIBUTE BY` clause, not just the tree root's.
/// OpenTenBase itself has no simple single-container way to stand up (its
/// own docs only cover a multi-node `pgxc_ctl deploy`/`init` cluster built
/// from source), so this runs the actual, unmodified
/// `TABLE_DISTRIBUTION_FOR_RELATIONS_SQL` query against a real PostgreSQL
/// container with a stand-in `pg_catalog.pgxc_class` table shaped like
/// OpenTenBase's real one — exercising the real SQL (join, `unnest($1::
/// text[], $2::text[])` parameter binding, casts, row parsing), not just the
/// clause-insertion string logic around it.
#[tokio::test]
async fn opentenbase_tree_ddl_gives_every_partition_its_own_distribute_by() {
let Some(container) =
start_docker_postgres_with_server_args("dbx-opentenbase-partition-ddl", &["-c", "allow_system_table_mods=on"])
else {
return;
};
psql(
&container,
"CREATE TABLE pg_catalog.pgxc_class (pcrelid oid PRIMARY KEY, pclocatortype \"char\", discolnums smallint[])",
);
psql(&container, "CREATE TABLE t (id integer NOT NULL, region text NOT NULL) PARTITION BY RANGE (id)");
psql(&container, "CREATE TABLE t_p1 PARTITION OF t FOR VALUES FROM (0) TO (100)");
psql(&container, "CREATE TABLE t_p2 PARTITION OF t FOR VALUES FROM (100) TO (200)");
// Root distributes by `region` (attnum 2); t_p1 distributes by `id`
// (attnum 1) instead, to prove each relation's own policy is looked up
// independently rather than the root's being reused for every relation.
// t_p2 deliberately has no pgxc_class row at all — it must be left
// without a DISTRIBUTE BY clause rather than erroring the whole DDL.
psql(&container, "INSERT INTO pg_catalog.pgxc_class VALUES ('t'::regclass, 'H', ARRAY[2]::smallint[])");
psql(&container, "INSERT INTO pg_catalog.pgxc_class VALUES ('t_p1'::regclass, 'H', ARRAY[1]::smallint[])");
let url = format!("postgres://postgres:postgres@127.0.0.1:{}/postgres", container.port);
let pool = dbx_core::db::postgres::connect(&url, std::time::Duration::from_secs(5)).await.expect("connect");
let distributions = dbx_core::db::opentenbase::table_distribution_for_relations(
&pool,
&[("public".to_string(), "t".to_string()), ("public".to_string(), "t_p1".to_string())],
)
.await
.expect("batched distribution query");
assert_eq!(
distributions.get(&("public".to_string(), "t".to_string())),
Some(&dbx_core::db::opentenbase::DistributionPolicy::Hash(vec!["region".to_string()]))
);
assert_eq!(
distributions.get(&("public".to_string(), "t_p1".to_string())),
Some(&dbx_core::db::opentenbase::DistributionPolicy::Hash(vec!["id".to_string()]))
);
let ddl = dbx_core::schema::opentenbase_ddl(&pool, "public", "t", true).await.expect("opentenbase tree ddl");
assert!(
ddl.contains("\"t\" (\n \"id\" integer NOT NULL,\n \"region\" text NOT NULL\n) PARTITION BY RANGE (id)\nDISTRIBUTE BY HASH (\"region\");"),
"root should get its own DISTRIBUTE BY HASH (region): {ddl}"
);
assert!(
ddl.contains(
"\"t_p1\" PARTITION OF \"public\".\"t\" FOR VALUES FROM (0) TO (100)\nDISTRIBUTE BY HASH (\"id\");"
),
"t_p1 should get its own DISTRIBUTE BY HASH (id), independent of the root's: {ddl}"
);
assert!(
ddl.contains("\"t_p2\" PARTITION OF \"public\".\"t\" FOR VALUES FROM (100) TO (200);"),
"t_p2 has no pgxc_class row and must not get a DISTRIBUTE BY clause: {ddl}"
);
assert_eq!(ddl.matches("DISTRIBUTE BY").count(), 2);
}
@@ -1,147 +1,12 @@
//! 整库 SQL 导出的端到端回归测试(元数据并发预取后输出必须与逐表查询一致)。
//! 需要本机 Docker;不可用时自动跳过。
use std::process::Command;
mod support;
use dbx_core::connection::AppState;
use dbx_core::database_export::{export_database_sql_core, DatabaseExportRequest};
use dbx_core::models::connection::{ConnectionConfig, DatabaseType};
use dbx_core::storage::Storage;
struct DockerPostgres {
name: String,
port: u16,
}
impl Drop for DockerPostgres {
fn drop(&mut self) {
let _ = Command::new("docker").args(["rm", "-f", &self.name]).status();
}
}
fn docker_ready() -> bool {
Command::new("docker")
.args(["version", "--format", "{{.Server.Version}}"])
.output()
.map(|output| output.status.success())
.unwrap_or(false)
}
fn start_docker_postgres() -> Option<DockerPostgres> {
if !docker_ready() {
eprintln!("skipping docker-backed export test because Docker is unavailable");
return None;
}
let port = portpicker::pick_unused_port().expect("pick unused postgres port");
let container = DockerPostgres { name: format!("dbx-export-prefetch-{}", uuid::Uuid::new_v4()), port };
let status = Command::new("docker")
.args([
"run",
"-d",
"--rm",
"--name",
&container.name,
"-e",
"POSTGRES_PASSWORD=postgres",
"-e",
"POSTGRES_USER=postgres",
"-e",
"POSTGRES_DB=postgres",
"-p",
&format!("{port}:5432"),
"postgres:16-alpine",
])
.status()
.expect("start docker postgres");
assert!(status.success(), "docker run postgres container should succeed");
// postgres 镜像 initdb 期间会短暂拉起再重启服务,单次探测可能命中引导期,
// 要求 SELECT 1 连续两次成功才算就绪
let mut consecutive_ok = 0;
for _ in 0..120 {
let ready = Command::new("docker")
.args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-c", "SELECT 1"])
.output()
.map(|output| output.status.success())
.unwrap_or(false);
consecutive_ok = if ready { consecutive_ok + 1 } else { 0 };
if consecutive_ok >= 2 {
return Some(container);
}
std::thread::sleep(std::time::Duration::from_millis(500));
}
panic!("postgres container did not become ready");
}
fn psql(container: &DockerPostgres, sql: &str) {
let output = Command::new("docker")
.args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-c", sql])
.output()
.expect("run psql");
assert!(output.status.success(), "psql failed: {}", String::from_utf8_lossy(&output.stderr));
}
fn postgres_test_config(id: &str, port: u16) -> ConnectionConfig {
ConnectionConfig {
docs_notes_path: None,
id: id.to_string(),
name: id.to_string(),
note: String::new(),
db_type: DatabaseType::Postgres,
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "127.0.0.1".to_string(),
port,
username: "postgres".to_string(),
password: "postgres".to_string(),
database: Some("postgres".to_string()),
default_schema: None,
visible_databases: None,
visible_schemas: None,
attached_databases: Vec::new(),
init_script: None,
color: None,
transport_layers: Vec::new(),
connect_timeout_secs: 5,
query_timeout_secs: 30,
idle_timeout_secs: 60,
keepalive_interval_secs: 0,
ssl: false,
ca_cert_path: String::new(),
client_cert_path: String::new(),
client_key_path: String::new(),
sysdba: false,
oracle_connection_type: None,
connection_string: None,
redis_connection_mode: None,
redis_sentinel_master: String::new(),
redis_sentinel_nodes: String::new(),
redis_sentinel_username: String::new(),
redis_sentinel_password: String::new(),
redis_sentinel_tls: false,
redis_cluster_nodes: String::new(),
redis_key_separator: dbx_core::models::connection::default_redis_key_separator(),
redis_scan_page_size: None,
redis_database_aliases: Default::default(),
etcd_endpoints: String::new(),
gbase_server: String::new(),
informix_server: String::new(),
external_config: None,
jdbc_driver_class: None,
jdbc_driver_paths: Vec::new(),
one_time: false,
save_password: true,
read_only: false,
is_production: false,
production_databases: vec![],
show_system_schemas: false,
database_info: None,
}
}
use support::{postgres_test_config, psql, start_docker_postgres};
#[test]
fn database_export_writes_structure_and_data_for_all_tables() {
@@ -162,7 +27,7 @@ fn database_export_writes_structure_and_data_for_all_tables() {
}
async fn run_database_export_writes_structure_and_data_for_all_tables() {
let Some(container) = start_docker_postgres() else {
let Some(container) = start_docker_postgres("dbx-export-prefetch") else {
return;
};
+191
View File
@@ -0,0 +1,191 @@
//! Shared Docker-backed PostgreSQL test scaffolding for integration tests
//! that need a real, disposable server (not a mock) to exercise driver SQL
//! against. Needs local Docker; callers should check `docker_ready()` (via
//! `start_docker_postgres`, which does this for them) and skip cleanly when
//! it's unavailable rather than failing the whole test run.
use std::process::Command;
use dbx_core::models::connection::{ConnectionConfig, DatabaseType};
pub struct DockerPostgres {
pub name: String,
pub port: u16,
/// `false` when this wraps an already-running container the caller
/// provided via `DBX_TEST_POSTGRES_CONTAINER` (see
/// `start_docker_postgres_with_server_args`) instead of one this helper
/// started itself — that container is the caller's to manage, not ours
/// to tear down.
remove_on_drop: bool,
}
impl Drop for DockerPostgres {
fn drop(&mut self) {
if self.remove_on_drop {
let _ = Command::new("docker").args(["rm", "-f", &self.name]).status();
}
}
}
pub fn docker_ready() -> bool {
Command::new("docker")
.args(["version", "--format", "{{.Server.Version}}"])
.output()
.map(|output| output.status.success())
.unwrap_or(false)
}
/// Starts a disposable `postgres:16-alpine` container named
/// `{name_prefix}-{random uuid}`, waiting for it to accept connections.
/// Returns `None` (rather than panicking) when Docker itself isn't
/// available, so tests can skip cleanly in environments without it.
pub fn start_docker_postgres(name_prefix: &str) -> Option<DockerPostgres> {
start_docker_postgres_with_server_args(name_prefix, &[])
}
/// Same as `start_docker_postgres`, but forwards extra `-c name=value`
/// postmaster arguments — e.g. `&["-c", "allow_system_table_mods=on"]` to let
/// a test create a stand-in `pg_catalog` table for a system catalog this
/// image doesn't actually have (like OpenTenBase/Cloudberry-only ones).
///
/// If `DBX_TEST_POSTGRES_CONTAINER`/`DBX_TEST_POSTGRES_PORT` are set, reuses
/// that already-running container instead of starting (and later tearing
/// down) a fresh one — handy for iterating on a single test locally without
/// paying container startup cost every run. `server_args` is ignored in that
/// case, since the caller's container is already configured however they set
/// it up.
pub fn start_docker_postgres_with_server_args(name_prefix: &str, server_args: &[&str]) -> Option<DockerPostgres> {
if let Ok(name) = std::env::var("DBX_TEST_POSTGRES_CONTAINER") {
let port = std::env::var("DBX_TEST_POSTGRES_PORT")
.expect("DBX_TEST_POSTGRES_PORT must accompany DBX_TEST_POSTGRES_CONTAINER")
.parse()
.expect("DBX_TEST_POSTGRES_PORT must be a valid port");
return Some(DockerPostgres { name, port, remove_on_drop: false });
}
if !docker_ready() {
eprintln!("skipping docker-backed test because Docker is unavailable");
return None;
}
let port = portpicker::pick_unused_port().expect("pick unused postgres port");
let container =
DockerPostgres { name: format!("{name_prefix}-{}", uuid::Uuid::new_v4()), port, remove_on_drop: true };
let mut args = vec![
"run".to_string(),
"-d".to_string(),
"--rm".to_string(),
"--name".to_string(),
container.name.clone(),
"-e".to_string(),
"POSTGRES_PASSWORD=postgres".to_string(),
"-e".to_string(),
"POSTGRES_USER=postgres".to_string(),
"-e".to_string(),
"POSTGRES_DB=postgres".to_string(),
"-p".to_string(),
format!("{port}:5432"),
"postgres:16-alpine".to_string(),
];
args.extend(server_args.iter().map(|arg| arg.to_string()));
let status = Command::new("docker").args(&args).status().expect("start docker postgres");
assert!(status.success(), "docker run postgres container should succeed");
// The postgres image's initdb briefly starts and restarts the server, so
// a single successful probe may still land inside that bootstrap window;
// require two consecutive successful `SELECT 1`s before calling it ready.
let mut consecutive_ok = 0;
for _ in 0..120 {
let ready = Command::new("docker")
.args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-c", "SELECT 1"])
.output()
.map(|output| output.status.success())
.unwrap_or(false);
consecutive_ok = if ready { consecutive_ok + 1 } else { 0 };
if consecutive_ok >= 2 {
return Some(container);
}
std::thread::sleep(std::time::Duration::from_millis(500));
}
panic!("postgres container did not become ready");
}
pub fn psql(container: &DockerPostgres, sql: &str) {
let output = Command::new("docker")
.args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-c", sql])
.output()
.expect("run psql");
assert!(output.status.success(), "psql failed: {}", String::from_utf8_lossy(&output.stderr));
}
// Each `tests/*.rs` file compiles this module as its own separate crate, so
// a helper only some of those test binaries call is "unused" in the others.
#[allow(dead_code)]
pub fn psql_allow_failure(container: &DockerPostgres, sql: &str) -> bool {
Command::new("docker")
.args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-c", sql])
.output()
.map(|output| output.status.success())
.unwrap_or(false)
}
pub fn postgres_test_config(id: &str, port: u16) -> ConnectionConfig {
ConnectionConfig {
docs_notes_path: None,
id: id.to_string(),
name: id.to_string(),
note: String::new(),
db_type: DatabaseType::Postgres,
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "127.0.0.1".to_string(),
port,
username: "postgres".to_string(),
password: "postgres".to_string(),
database: Some("postgres".to_string()),
default_schema: None,
visible_databases: None,
visible_schemas: None,
attached_databases: Vec::new(),
init_script: None,
color: None,
transport_layers: Vec::new(),
connect_timeout_secs: 5,
query_timeout_secs: 30,
idle_timeout_secs: 60,
keepalive_interval_secs: 0,
ssl: false,
ca_cert_path: String::new(),
client_cert_path: String::new(),
client_key_path: String::new(),
sysdba: false,
oracle_connection_type: None,
connection_string: None,
redis_connection_mode: None,
redis_sentinel_master: String::new(),
redis_sentinel_nodes: String::new(),
redis_sentinel_username: String::new(),
redis_sentinel_password: String::new(),
redis_sentinel_tls: false,
redis_cluster_nodes: String::new(),
redis_key_separator: dbx_core::models::connection::default_redis_key_separator(),
redis_scan_page_size: None,
redis_database_aliases: Default::default(),
etcd_endpoints: String::new(),
gbase_server: String::new(),
informix_server: String::new(),
external_config: None,
jdbc_driver_class: None,
jdbc_driver_paths: Vec::new(),
one_time: false,
save_password: true,
read_only: false,
is_production: false,
production_databases: vec![],
show_system_schemas: false,
database_info: None,
}
}