fix(postgres): correct partition DDL generation
This commit is contained in:
1 parent
e9ca697c3b
commit
d085c5b4f4
9 files changed
+960
-378
No files matched your search
@@ -170,10 +170,14 @@ fn render_external_table_ddl(ddl: &str, external: &ExternalTableDefinition) -> R
|
|||||||
.ok_or_else(|| "Cloudberry fallback DDL has no CREATE TABLE terminator".to_string())?;
|
.ok_or_else(|| "Cloudberry fallback DDL has no CREATE TABLE terminator".to_string())?;
|
||||||
// When the fallback DDL already read `CREATE FOREIGN TABLE`, the generic
|
// When the fallback DDL already read `CREATE FOREIGN TABLE`, the generic
|
||||||
// renderer also already appended its own ` SERVER ... [OPTIONS (...)]`
|
// renderer also already appended its own ` SERVER ... [OPTIONS (...)]`
|
||||||
// clause (from the same relkind='f' catalog data). Drop that so this
|
// clause (from the same relkind='f' catalog data) right after the column
|
||||||
// doesn't emit the clause twice — Cloudberry's own `external` definition
|
// list's closing paren. Drop that so this doesn't emit the clause twice —
|
||||||
// above is authoritative here.
|
// Cloudberry's own `external` definition above is authoritative here.
|
||||||
let body_end = ddl[header_len..insertion].find(" SERVER \"").map(|pos| header_len + pos).unwrap_or(insertion);
|
// Locate the column list's own closing paren structurally (tracking
|
||||||
|
// parenthesis depth and skipping quoted content) instead of searching for
|
||||||
|
// the literal text `" SERVER \""`, which a column's DEFAULT/CHECK
|
||||||
|
// expression could contain and match spuriously.
|
||||||
|
let body_end = super::ddl_scan::find_top_level_paren_close(ddl, header_len).unwrap_or(insertion);
|
||||||
let mut output = String::with_capacity(ddl.len() + external.options.len() * 24 + 48);
|
let mut output = String::with_capacity(ddl.len() + external.options.len() * 24 + 48);
|
||||||
output.push_str("CREATE FOREIGN TABLE ");
|
output.push_str("CREATE FOREIGN TABLE ");
|
||||||
output.push_str(&ddl[header_len..body_end]);
|
output.push_str(&ddl[header_len..body_end]);
|
||||||
@@ -364,6 +368,34 @@ mod tests {
|
|||||||
assert!(!rendered.contains("stale_server"));
|
assert!(!rendered.contains("stale_server"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn renders_external_table_when_a_column_default_contains_literal_server_text() {
|
||||||
|
// A naive text search for `" SERVER \""` to find the stale SERVER
|
||||||
|
// clause's start would match this DEFAULT's embedded text instead of
|
||||||
|
// the real trailing SERVER clause, truncating the column list.
|
||||||
|
let ddl = "CREATE FOREIGN TABLE \"public\".\"external_events\" (\n \"id\" integer,\n \"note\" text DEFAULT 'contact SERVER \"admin\"'\n) SERVER \"stale_server\";\n";
|
||||||
|
let rendered = append_table_modifiers(
|
||||||
|
ddl,
|
||||||
|
&TableModifiers {
|
||||||
|
external: Some(ExternalTableDefinition {
|
||||||
|
server: "gp_exttable_server".to_string(),
|
||||||
|
options: vec!["format=csv".to_string()],
|
||||||
|
}),
|
||||||
|
..modifiers(None)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(rendered.contains("\"note\" text DEFAULT 'contact SERVER \"admin\"'"));
|
||||||
|
assert_eq!(
|
||||||
|
rendered.matches("SERVER").count(),
|
||||||
|
2,
|
||||||
|
"expected exactly the embedded literal plus the real clause: {rendered}"
|
||||||
|
);
|
||||||
|
assert!(rendered.contains("SERVER \"gp_exttable_server\""));
|
||||||
|
assert!(!rendered.contains("stale_server"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn ddl_queries_avoid_privileged_external_catalog_view() {
|
fn ddl_queries_avoid_privileged_external_catalog_view() {
|
||||||
assert_eq!(CLOUD_BERRY_TABLE_DDL_SQL, "SELECT pg_get_tabledef($1, $2, true)");
|
assert_eq!(CLOUD_BERRY_TABLE_DDL_SQL, "SELECT pg_get_tabledef($1, $2, true)");
|
||||||
|
|||||||
@@ -0,0 +1,223 @@
|
|||||||
|
//! Structural, quote-and-parenthesis-aware scanning helpers for
|
||||||
|
//! post-processing already-rendered `CREATE TABLE` DDL text.
|
||||||
|
//!
|
||||||
|
//! A naive `str::find` for a literal marker (e.g. `" SERVER \""` or a bare
|
||||||
|
//! `;`) can be fooled by that same text appearing inside a quoted string
|
||||||
|
//! literal, a quoted identifier, or a nested parenthesized sub-expression —
|
||||||
|
//! a column's `DEFAULT`/`CHECK` expression can legally contain any of those.
|
||||||
|
//! These helpers track quote and parenthesis depth so callers only ever
|
||||||
|
//! match at the top level.
|
||||||
|
|
||||||
|
use std::ops::Range;
|
||||||
|
|
||||||
|
fn char_at(sql: &str, pos: usize) -> Option<char> {
|
||||||
|
sql.get(pos..)?.chars().next()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Skips a `'...'` string literal (`''`-escaped, with backslash-escape
|
||||||
|
/// tolerance) or a `"..."` quoted identifier (`""`-escaped) starting at
|
||||||
|
/// `pos`, which must point at the opening `quote` character. Returns the
|
||||||
|
/// index just past the closing quote (or `sql.len()` if unterminated).
|
||||||
|
fn skip_quoted(sql: &str, pos: usize, quote: char) -> usize {
|
||||||
|
let mut i = pos + quote.len_utf8();
|
||||||
|
while i < sql.len() {
|
||||||
|
let Some(ch) = char_at(sql, i) else { break };
|
||||||
|
let next = char_at(sql, i + ch.len_utf8());
|
||||||
|
if ch == quote {
|
||||||
|
if next == Some(quote) {
|
||||||
|
i += ch.len_utf8() + quote.len_utf8();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return i + ch.len_utf8();
|
||||||
|
}
|
||||||
|
if quote == '\'' && ch == '\\' {
|
||||||
|
i += ch.len_utf8();
|
||||||
|
if let Some(escaped) = char_at(sql, i) {
|
||||||
|
i += escaped.len_utf8();
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
i += ch.len_utf8();
|
||||||
|
}
|
||||||
|
sql.len()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Skips a PostgreSQL `$tag$...$tag$` dollar-quoted body starting at `pos`
|
||||||
|
/// (pointing at the first `$`). Returns `None` if `pos` isn't actually the
|
||||||
|
/// start of a valid dollar-quote tag (e.g. a stray `$` in an expression), in
|
||||||
|
/// which case the caller should treat `$` as an ordinary character.
|
||||||
|
fn skip_dollar_quoted(sql: &str, pos: usize) -> Option<usize> {
|
||||||
|
let tag_end_offset = sql.get(pos + 1..)?.find('$')?;
|
||||||
|
let tag_end = pos + 1 + tag_end_offset;
|
||||||
|
let tag = &sql[pos + 1..tag_end];
|
||||||
|
let valid_tag = tag.is_empty()
|
||||||
|
|| (tag.chars().next().is_some_and(|ch| ch.is_ascii_alphabetic() || ch == '_')
|
||||||
|
&& tag.chars().all(|ch| ch.is_ascii_alphanumeric() || ch == '_'));
|
||||||
|
if !valid_tag {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let delimiter = &sql[pos..=tag_end];
|
||||||
|
let content_start = tag_end + 1;
|
||||||
|
sql.get(content_start..)?.find(delimiter).map(|offset| content_start + offset + delimiter.len())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Finds the byte index just past the first top-level (parenthesis-depth-0)
|
||||||
|
/// closing paren at or after `from` — the end of the outermost parenthesized
|
||||||
|
/// group that starts somewhere at or after `from` — treating quoted or
|
||||||
|
/// dollar-quoted content as opaque so parens or keywords inside a
|
||||||
|
/// `DEFAULT`/`CHECK` expression can't confuse the count. Returns `None` if no
|
||||||
|
/// top-level `(` is ever opened, or it's never closed.
|
||||||
|
pub(crate) fn find_top_level_paren_close(sql: &str, from: usize) -> Option<usize> {
|
||||||
|
let mut i = from;
|
||||||
|
let mut depth: usize = 0;
|
||||||
|
let mut opened = false;
|
||||||
|
while i < sql.len() {
|
||||||
|
let ch = char_at(sql, i)?;
|
||||||
|
match ch {
|
||||||
|
'\'' | '"' => {
|
||||||
|
i = skip_quoted(sql, i, ch);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
'$' => {
|
||||||
|
if let Some(end) = skip_dollar_quoted(sql, i) {
|
||||||
|
i = end;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
'(' => {
|
||||||
|
depth += 1;
|
||||||
|
opened = true;
|
||||||
|
}
|
||||||
|
')' => {
|
||||||
|
depth = depth.saturating_sub(1);
|
||||||
|
i += ch.len_utf8();
|
||||||
|
if opened && depth == 0 {
|
||||||
|
return Some(i);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
i += ch.len_utf8();
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Finds the byte index of the next top-level (depth-0, outside any quoted
|
||||||
|
/// or dollar-quoted content) `;` at or after `from`.
|
||||||
|
fn find_top_level_semicolon(sql: &str, from: usize) -> Option<usize> {
|
||||||
|
let mut i = from;
|
||||||
|
let mut depth: usize = 0;
|
||||||
|
while i < sql.len() {
|
||||||
|
let ch = char_at(sql, i)?;
|
||||||
|
match ch {
|
||||||
|
'\'' | '"' => {
|
||||||
|
i = skip_quoted(sql, i, ch);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
'$' => {
|
||||||
|
if let Some(end) = skip_dollar_quoted(sql, i) {
|
||||||
|
i = end;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
'(' => depth += 1,
|
||||||
|
')' => depth = depth.saturating_sub(1),
|
||||||
|
';' if depth == 0 => return Some(i),
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
i += ch.len_utf8();
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Splits `sql` into consecutive top-level statement ranges, each ending
|
||||||
|
/// just past its own top-level `;`. The last statement may lack a trailing
|
||||||
|
/// `;`, in which case its range runs to the end of the string.
|
||||||
|
pub(crate) fn top_level_statement_ranges(sql: &str) -> Vec<Range<usize>> {
|
||||||
|
let mut ranges = Vec::new();
|
||||||
|
let mut start = 0;
|
||||||
|
while start < sql.len() {
|
||||||
|
match find_top_level_semicolon(sql, start) {
|
||||||
|
Some(semi) => {
|
||||||
|
ranges.push(start..semi + 1);
|
||||||
|
start = semi + 1;
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
ranges.push(start..sql.len());
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ranges
|
||||||
|
}
|
||||||
|
|
||||||
|
/// If `stmt` (ignoring leading whitespace) begins with `CREATE TABLE
|
||||||
|
/// "schema"."table"` or `CREATE FOREIGN TABLE "schema"."table"`, returns the
|
||||||
|
/// unescaped `(schema, table)` pair.
|
||||||
|
pub(crate) fn parse_create_table_relation(stmt: &str) -> Option<(String, String)> {
|
||||||
|
let trimmed = stmt.trim_start();
|
||||||
|
let rest =
|
||||||
|
trimmed.strip_prefix("CREATE FOREIGN TABLE ").or_else(|| trimmed.strip_prefix("CREATE TABLE "))?.trim_start();
|
||||||
|
let (schema, rest) = unquote_ident(rest)?;
|
||||||
|
let (table, _) = unquote_ident(rest.strip_prefix('.')?)?;
|
||||||
|
Some((schema, table))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unquote_ident(s: &str) -> Option<(String, &str)> {
|
||||||
|
if !s.starts_with('"') {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let end = skip_quoted(s, 0, '"');
|
||||||
|
if end < 2 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let raw = &s[1..end - 1];
|
||||||
|
Some((raw.replace("\"\"", "\""), &s[end..]))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn paren_close_skips_nested_parens_and_quotes() {
|
||||||
|
let ddl =
|
||||||
|
"\"public\".\"t\" (\n \"a\" text DEFAULT 'x(y)'::text,\n \"b\" text CHECK ((b <> 'z'))\n) SERVER \"s\"";
|
||||||
|
let close = find_top_level_paren_close(ddl, 0).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
&ddl[..close],
|
||||||
|
"\"public\".\"t\" (\n \"a\" text DEFAULT 'x(y)'::text,\n \"b\" text CHECK ((b <> 'z'))\n)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn paren_close_ignores_literal_server_text_inside_quotes() {
|
||||||
|
let ddl = "\"public\".\"t\" (\n \"note\" text DEFAULT 'contact SERVER \"admin\"'\n) SERVER \"real_server\"";
|
||||||
|
let close = find_top_level_paren_close(ddl, 0).unwrap();
|
||||||
|
assert!(ddl[close..].starts_with(" SERVER \"real_server\""));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn top_level_statement_ranges_split_on_depth_zero_semicolons() {
|
||||||
|
let ddl = "CREATE TABLE \"s\".\"t\" (\n \"a\" text DEFAULT 'x;y'\n);\nALTER TABLE \"s\".\"t\" ADD CONSTRAINT c CHECK (a <> '');";
|
||||||
|
let ranges = top_level_statement_ranges(ddl);
|
||||||
|
assert_eq!(ranges.len(), 2);
|
||||||
|
assert!(ddl[ranges[0].clone()].starts_with("CREATE TABLE"));
|
||||||
|
assert!(ddl[ranges[0].clone()].ends_with(";"));
|
||||||
|
assert!(ddl[ranges[1].clone()].trim_start().starts_with("ALTER TABLE"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_create_table_relation_unescapes_doubled_quotes() {
|
||||||
|
let (schema, table) =
|
||||||
|
parse_create_table_relation("CREATE TABLE \"pub\"\"lic\".\"t\" (\n \"a\" int\n);").unwrap();
|
||||||
|
assert_eq!(schema, "pub\"lic");
|
||||||
|
assert_eq!(table, "t");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_create_table_relation_rejects_non_create_table() {
|
||||||
|
assert!(parse_create_table_relation("ALTER TABLE \"s\".\"t\" ADD CONSTRAINT c CHECK (true);").is_none());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,6 +3,7 @@ pub mod clickhouse_driver;
|
|||||||
pub mod cloudberry;
|
pub mod cloudberry;
|
||||||
pub mod cloudflare_d1;
|
pub mod cloudflare_d1;
|
||||||
pub use cloudflare_d1 as cloudflare_d1_driver;
|
pub use cloudflare_d1 as cloudflare_d1_driver;
|
||||||
|
pub(crate) mod ddl_scan;
|
||||||
pub mod document_result;
|
pub mod document_result;
|
||||||
pub mod dolt;
|
pub mod dolt;
|
||||||
pub mod doris;
|
pub mod doris;
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
use deadpool_postgres::Pool;
|
use deadpool_postgres::Pool;
|
||||||
|
|
||||||
use crate::db;
|
use crate::db;
|
||||||
@@ -20,6 +22,24 @@ const TABLE_DISTRIBUTION_SQL: &str = "SELECT c.pclocatortype::text, \
|
|||||||
WHERE namespace.nspname = $1 AND relation.relname = $2 \
|
WHERE namespace.nspname = $1 AND relation.relname = $2 \
|
||||||
LIMIT 1";
|
LIMIT 1";
|
||||||
|
|
||||||
|
/// Batched sibling of `TABLE_DISTRIBUTION_SQL`, for appending each
|
||||||
|
/// partition's own `DISTRIBUTE BY` clause when rendering a whole partition
|
||||||
|
/// tree instead of just the root relation.
|
||||||
|
const TABLE_DISTRIBUTION_FOR_RELATIONS_SQL: &str =
|
||||||
|
"SELECT namespace.nspname, relation.relname, c.pclocatortype::text, \
|
||||||
|
COALESCE(( \
|
||||||
|
SELECT string_agg(a.attname, E'\\n' ORDER BY distributed_column.ordinality) \
|
||||||
|
FROM unnest(c.discolnums::smallint[]) WITH ORDINALITY \
|
||||||
|
AS distributed_column(attnum, ordinality) \
|
||||||
|
JOIN pg_catalog.pg_attribute a \
|
||||||
|
ON a.attrelid = c.pcrelid AND a.attnum = distributed_column.attnum \
|
||||||
|
WHERE NOT a.attisdropped \
|
||||||
|
), '')::text \
|
||||||
|
FROM unnest($1::text[], $2::text[]) AS rel(rel_schema, rel_table) \
|
||||||
|
JOIN pg_catalog.pg_namespace namespace ON namespace.nspname = rel.rel_schema \
|
||||||
|
JOIN pg_catalog.pg_class relation ON relation.relnamespace = namespace.oid AND relation.relname = rel.rel_table \
|
||||||
|
JOIN pg_catalog.pgxc_class c ON c.pcrelid = relation.oid";
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub enum DistributionPolicy {
|
pub enum DistributionPolicy {
|
||||||
Hash(Vec<String>),
|
Hash(Vec<String>),
|
||||||
@@ -52,6 +72,43 @@ pub async fn table_distribution(pool: &Pool, schema: &str, table: &str) -> Resul
|
|||||||
parse_distribution_policy(&locator_type, columns)
|
parse_distribution_policy(&locator_type, columns)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Batched sibling of `table_distribution`, for a whole partition tree.
|
||||||
|
/// Relations without a distribution policy (or that don't exist) are simply
|
||||||
|
/// absent from the result map rather than erroring the whole batch.
|
||||||
|
pub async fn table_distribution_for_relations(
|
||||||
|
pool: &Pool,
|
||||||
|
relations: &[(String, String)],
|
||||||
|
) -> Result<HashMap<(String, String), DistributionPolicy>, String> {
|
||||||
|
if relations.is_empty() {
|
||||||
|
return Ok(HashMap::new());
|
||||||
|
}
|
||||||
|
let client = db::postgres::checkout_postgres_client(pool, None, super::connection_timeout()).await?;
|
||||||
|
let schemas: Vec<&str> = relations.iter().map(|(schema, _)| schema.as_str()).collect();
|
||||||
|
let tables: Vec<&str> = relations.iter().map(|(_, table)| table.as_str()).collect();
|
||||||
|
let rows = client
|
||||||
|
.query(TABLE_DISTRIBUTION_FOR_RELATIONS_SQL, &[&schemas, &tables])
|
||||||
|
.await
|
||||||
|
.map_err(|error| error.to_string())?;
|
||||||
|
let mut result = HashMap::new();
|
||||||
|
for row in &rows {
|
||||||
|
let schema = row.try_get::<_, String>(0).map_err(|error| error.to_string())?;
|
||||||
|
let table = row.try_get::<_, String>(1).map_err(|error| error.to_string())?;
|
||||||
|
let locator_type = row.try_get::<_, String>(2).map_err(|error| error.to_string())?;
|
||||||
|
let columns = row
|
||||||
|
.try_get::<_, String>(3)
|
||||||
|
.map_err(|error| error.to_string())?
|
||||||
|
.lines()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|column| !column.is_empty())
|
||||||
|
.map(str::to_string)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if let Some(distribution) = parse_distribution_policy(&locator_type, columns)? {
|
||||||
|
result.insert((schema, table), distribution);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn append_distribution_clause(ddl: &str, distribution: &DistributionPolicy) -> Result<String, String> {
|
pub fn append_distribution_clause(ddl: &str, distribution: &DistributionPolicy) -> Result<String, String> {
|
||||||
let clause = render_distribution_clause(distribution)?;
|
let clause = render_distribution_clause(distribution)?;
|
||||||
let insertion = ddl
|
let insertion = ddl
|
||||||
@@ -66,6 +123,41 @@ pub fn append_distribution_clause(ddl: &str, distribution: &DistributionPolicy)
|
|||||||
Ok(output)
|
Ok(output)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Tree-aware sibling of `append_distribution_clause`: `ddl` may contain
|
||||||
|
/// several relations' worth of statements concatenated together (root plus
|
||||||
|
/// every partition), so each `CREATE [FOREIGN] TABLE "schema"."table"`
|
||||||
|
/// statement gets its own `DISTRIBUTE BY` clause inserted before its own
|
||||||
|
/// terminator, looked up by that statement's own `(schema, table)` — instead
|
||||||
|
/// of inserting a single clause at the first `;` in the whole string, which
|
||||||
|
/// would only ever land on the root relation's statement.
|
||||||
|
pub fn append_distribution_clauses(
|
||||||
|
ddl: &str,
|
||||||
|
distributions: &HashMap<(String, String), DistributionPolicy>,
|
||||||
|
) -> Result<String, String> {
|
||||||
|
if distributions.is_empty() {
|
||||||
|
return Ok(ddl.to_string());
|
||||||
|
}
|
||||||
|
let mut output = String::with_capacity(ddl.len() + distributions.len() * 32);
|
||||||
|
let mut cursor = 0usize;
|
||||||
|
for range in db::ddl_scan::top_level_statement_ranges(ddl) {
|
||||||
|
let statement = &ddl[range.clone()];
|
||||||
|
let Some(relation) = db::ddl_scan::parse_create_table_relation(statement) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let Some(distribution) = distributions.get(&relation) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let clause = render_distribution_clause(distribution)?;
|
||||||
|
let insertion = if statement.ends_with(';') { range.end - 1 } else { range.end };
|
||||||
|
output.push_str(&ddl[cursor..insertion]);
|
||||||
|
output.push('\n');
|
||||||
|
output.push_str(&clause);
|
||||||
|
cursor = insertion;
|
||||||
|
}
|
||||||
|
output.push_str(&ddl[cursor..]);
|
||||||
|
Ok(output)
|
||||||
|
}
|
||||||
|
|
||||||
fn parse_distribution_policy(locator_type: &str, columns: Vec<String>) -> Result<Option<DistributionPolicy>, String> {
|
fn parse_distribution_policy(locator_type: &str, columns: Vec<String>) -> Result<Option<DistributionPolicy>, String> {
|
||||||
let column_distribution = |kind| {
|
let column_distribution = |kind| {
|
||||||
if columns.is_empty() {
|
if columns.is_empty() {
|
||||||
@@ -148,4 +240,32 @@ mod tests {
|
|||||||
assert!(TABLE_DISTRIBUTION_SQL.contains("WITH ORDINALITY"));
|
assert!(TABLE_DISTRIBUTION_SQL.contains("WITH ORDINALITY"));
|
||||||
assert!(TABLE_DISTRIBUTION_SQL.contains("NOT a.attisdropped"));
|
assert!(TABLE_DISTRIBUTION_SQL.contains("NOT a.attisdropped"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn batched_distribution_query_matches_opentenbase_catalog_contract() {
|
||||||
|
assert!(TABLE_DISTRIBUTION_FOR_RELATIONS_SQL.contains("unnest($1::text[], $2::text[])"));
|
||||||
|
assert!(TABLE_DISTRIBUTION_FOR_RELATIONS_SQL.contains("pg_catalog.pgxc_class"));
|
||||||
|
assert!(TABLE_DISTRIBUTION_FOR_RELATIONS_SQL.contains("WITH ORDINALITY"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn appends_each_partition_its_own_distribution_clause_in_a_multi_relation_tree() {
|
||||||
|
let ddl = "CREATE TABLE \"public\".\"events\" (\n \"id\" integer\n) PARTITION BY RANGE (id);\n\nCREATE TABLE \"public\".\"events_2026\" PARTITION OF \"public\".\"events\" FOR VALUES FROM (1) TO (100);\n\nCREATE TABLE \"public\".\"events_2027\" PARTITION OF \"public\".\"events\" FOR VALUES FROM (100) TO (200);\n";
|
||||||
|
let mut distributions = HashMap::new();
|
||||||
|
distributions
|
||||||
|
.insert(("public".to_string(), "events".to_string()), DistributionPolicy::Hash(vec!["id".to_string()]));
|
||||||
|
distributions.insert(("public".to_string(), "events_2027".to_string()), DistributionPolicy::Replication);
|
||||||
|
// "events_2026" deliberately has no entry — it must be left untouched.
|
||||||
|
|
||||||
|
let rendered = append_distribution_clauses(ddl, &distributions).unwrap();
|
||||||
|
|
||||||
|
assert!(rendered.contains("PARTITION BY RANGE (id)\nDISTRIBUTE BY HASH (\"id\");"));
|
||||||
|
assert!(rendered.contains(
|
||||||
|
"CREATE TABLE \"public\".\"events_2027\" PARTITION OF \"public\".\"events\" FOR VALUES FROM (100) TO (200)\nDISTRIBUTE BY REPLICATION;"
|
||||||
|
));
|
||||||
|
assert!(rendered.contains(
|
||||||
|
"CREATE TABLE \"public\".\"events_2026\" PARTITION OF \"public\".\"events\" FOR VALUES FROM (1) TO (100);"
|
||||||
|
));
|
||||||
|
assert_eq!(rendered.matches("DISTRIBUTE BY").count(), 2);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -1018,6 +1018,37 @@ fn pg_error_to_string(err: tokio_postgres::Error) -> String {
|
|||||||
err.as_db_error().map(ToString::to_string).unwrap_or_else(|| err.to_string())
|
err.as_db_error().map(ToString::to_string).unwrap_or_else(|| err.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Tries each SQL tier in `tiers` in order (most-capable first), via `run`,
|
||||||
|
/// returning the first tier that succeeds. Every driver-compat query in this
|
||||||
|
/// module (a "does this server have the newer catalog column" primary/compat
|
||||||
|
/// split, occasionally with a further information_schema fallback) used to
|
||||||
|
/// hand-roll this same try/log/combine-errors shape once per query; this is
|
||||||
|
/// the shared version.
|
||||||
|
///
|
||||||
|
/// If every tier fails, all of their errors are logged together at debug
|
||||||
|
/// level (so a fallback firing in production is diagnosable) and the last
|
||||||
|
/// tier's error is returned to the caller, since it's usually the most
|
||||||
|
/// specific one for whatever the connected server actually is.
|
||||||
|
async fn query_with_compat_fallback<T, F, Fut>(
|
||||||
|
log_context: &str,
|
||||||
|
tiers: &[&'static str],
|
||||||
|
mut run: F,
|
||||||
|
) -> Result<T, String>
|
||||||
|
where
|
||||||
|
F: FnMut(&'static str) -> Fut,
|
||||||
|
Fut: std::future::Future<Output = Result<T, tokio_postgres::Error>>,
|
||||||
|
{
|
||||||
|
let mut errors: Vec<String> = Vec::new();
|
||||||
|
for sql in tiers {
|
||||||
|
match run(sql).await {
|
||||||
|
Ok(value) => return Ok(value),
|
||||||
|
Err(error) => errors.push(pg_error_to_string(error)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log::debug!("[postgres][{log_context}:compat-failed] {}", errors.join("; "));
|
||||||
|
Err(errors.into_iter().next_back().unwrap_or_else(|| format!("[postgres][{log_context}] no SQL tiers configured")))
|
||||||
|
}
|
||||||
|
|
||||||
fn pg_db_error_to_string(err: &tokio_postgres::error::DbError) -> String {
|
fn pg_db_error_to_string(err: &tokio_postgres::error::DbError) -> String {
|
||||||
format!("{err} (SQLSTATE {})", err.code().code())
|
format!("{err} (SQLSTATE {})", err.code().code())
|
||||||
}
|
}
|
||||||
@@ -3243,23 +3274,11 @@ pub async fn get_columns_for_relations(
|
|||||||
) -> Result<HashMap<i64, Vec<ColumnInfo>>, String> {
|
) -> Result<HashMap<i64, Vec<ColumnInfo>>, String> {
|
||||||
let oids: Vec<i64> = relations.iter().map(|(oid, _, _)| *oid).collect();
|
let oids: Vec<i64> = relations.iter().map(|(oid, _, _)| *oid).collect();
|
||||||
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
|
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
|
||||||
let [primary_sql, compat_sql] = postgres_columns_for_relations_query_tiers();
|
let tiers = postgres_columns_for_relations_query_tiers();
|
||||||
match get_columns_for_relations_with_sql(&client, primary_sql, &oids).await {
|
query_with_compat_fallback("get_columns_for_relations", &tiers, |sql| {
|
||||||
Ok(columns) => Ok(columns),
|
get_columns_for_relations_with_sql(&client, sql, &oids)
|
||||||
Err(primary_error) => match get_columns_for_relations_with_sql(&client, compat_sql, &oids).await {
|
})
|
||||||
Ok(columns) => Ok(columns),
|
.await
|
||||||
Err(fallback_error) => {
|
|
||||||
let primary_message = pg_error_to_string(primary_error);
|
|
||||||
let fallback_message = pg_error_to_string(fallback_error);
|
|
||||||
log::debug!(
|
|
||||||
"[postgres][get_columns_for_relations:compat-failed] primary_error={} fallback_error={}",
|
|
||||||
primary_message,
|
|
||||||
fallback_message
|
|
||||||
);
|
|
||||||
Err(fallback_message)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn get_columns_for_relations_with_sql(
|
async fn get_columns_for_relations_with_sql(
|
||||||
@@ -3280,6 +3299,15 @@ fn postgres_columns_for_relations_query_tiers() -> [&'static str; 2] {
|
|||||||
[postgres_columns_for_relations_sql(), postgres_columns_for_relations_compat_sql()]
|
[postgres_columns_for_relations_sql(), postgres_columns_for_relations_compat_sql()]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Sibling of `POSTGRES_COLUMNS_SQL`/`POSTGRES_COLUMNS_COMPAT_SQL` below (~line
|
||||||
|
// 4880): same column list and detection logic (identity/serial inference,
|
||||||
|
// numeric precision/scale, enum values), batched by oid instead of a single
|
||||||
|
// (schema, table) pair. Kept as a separate literal rather than sharing a
|
||||||
|
// fragment — `pg_class` needs the `c` alias here for the oid filter, which
|
||||||
|
// pushes `information_schema.columns` to `ic` instead of the single-relation
|
||||||
|
// version's `c`, and every column's position in the SELECT list is relied on
|
||||||
|
// positionally by `column_info_from_row_offset`. A change to one almost
|
||||||
|
// certainly needs the same change in the other.
|
||||||
fn postgres_columns_for_relations_sql() -> &'static str {
|
fn postgres_columns_for_relations_sql() -> &'static str {
|
||||||
"SELECT c.oid::bigint AS relid, a.attname AS column_name, \
|
"SELECT c.oid::bigint AS relid, a.attname AS column_name, \
|
||||||
format_type(a.atttypid, a.atttypmod) AS full_type, \
|
format_type(a.atttypid, a.atttypmod) AS full_type, \
|
||||||
@@ -3352,6 +3380,8 @@ fn postgres_columns_for_relations_sql() -> &'static str {
|
|||||||
ORDER BY c.oid, a.attnum"
|
ORDER BY c.oid, a.attnum"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Compat-tier sibling of `POSTGRES_COLUMNS_COMPAT_SQL` (~line 4938) — see the
|
||||||
|
// note on `postgres_columns_for_relations_sql` above.
|
||||||
fn postgres_columns_for_relations_compat_sql() -> &'static str {
|
fn postgres_columns_for_relations_compat_sql() -> &'static str {
|
||||||
"SELECT c.oid::bigint AS relid, a.attname AS column_name, \
|
"SELECT c.oid::bigint AS relid, a.attname AS column_name, \
|
||||||
format_type(a.atttypid, a.atttypmod) AS full_type, \
|
format_type(a.atttypid, a.atttypmod) AS full_type, \
|
||||||
@@ -3438,23 +3468,11 @@ pub async fn list_indexes_for_relations(
|
|||||||
) -> Result<HashMap<i64, Vec<IndexInfo>>, String> {
|
) -> Result<HashMap<i64, Vec<IndexInfo>>, String> {
|
||||||
let oids: Vec<i64> = relations.iter().map(|(oid, _, _)| *oid).collect();
|
let oids: Vec<i64> = relations.iter().map(|(oid, _, _)| *oid).collect();
|
||||||
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
|
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
|
||||||
let [primary_sql, compat_sql] = postgres_indexes_for_relations_query_tiers();
|
let tiers = postgres_indexes_for_relations_query_tiers();
|
||||||
match list_indexes_for_relations_with_sql(&client, primary_sql, &oids).await {
|
query_with_compat_fallback("list_indexes_for_relations", &tiers, |sql| {
|
||||||
Ok(indexes) => Ok(indexes),
|
list_indexes_for_relations_with_sql(&client, sql, &oids)
|
||||||
Err(primary_error) => match list_indexes_for_relations_with_sql(&client, compat_sql, &oids).await {
|
})
|
||||||
Ok(indexes) => Ok(indexes),
|
.await
|
||||||
Err(fallback_error) => {
|
|
||||||
let primary_message = pg_error_to_string(primary_error);
|
|
||||||
let fallback_message = pg_error_to_string(fallback_error);
|
|
||||||
log::debug!(
|
|
||||||
"[postgres][list_indexes_for_relations:compat-failed] primary_error={} fallback_error={}",
|
|
||||||
primary_message,
|
|
||||||
fallback_message
|
|
||||||
);
|
|
||||||
Err(fallback_message)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_indexes_for_relations_with_sql(
|
async fn list_indexes_for_relations_with_sql(
|
||||||
@@ -3493,6 +3511,11 @@ fn postgres_indexes_for_relations_query_tiers() -> [&'static str; 2] {
|
|||||||
[postgres_indexes_for_relations_sql(), postgres_indexes_for_relations_compat_sql()]
|
[postgres_indexes_for_relations_sql(), postgres_indexes_for_relations_compat_sql()]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Sibling of `POSTGRES_INDEXES_SQL`/`POSTGRES_INDEXES_COMPAT_SQL` (~line
|
||||||
|
// 6042): same index-detection logic, batched by oid instead of a single
|
||||||
|
// (schema, table) pair. Not merged into a shared fragment for the same
|
||||||
|
// reason as the columns queries above — an alias would need renaming to
|
||||||
|
// line up, and result columns are read positionally.
|
||||||
fn postgres_indexes_for_relations_sql() -> &'static str {
|
fn postgres_indexes_for_relations_sql() -> &'static str {
|
||||||
"SELECT t.oid::bigint AS relid, i.relname AS index_name, \
|
"SELECT t.oid::bigint AS relid, i.relname AS index_name, \
|
||||||
array_agg(COALESCE(a.attname, pg_get_indexdef(ix.indexrelid, k.n::int, true)) ORDER BY k.n) AS columns, \
|
array_agg(COALESCE(a.attname, pg_get_indexdef(ix.indexrelid, k.n::int, true)) ORDER BY k.n) AS columns, \
|
||||||
@@ -3515,6 +3538,8 @@ fn postgres_indexes_for_relations_sql() -> &'static str {
|
|||||||
ORDER BY t.oid, i.relname"
|
ORDER BY t.oid, i.relname"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Compat-tier sibling of `POSTGRES_INDEXES_COMPAT_SQL` (~line 6063) — see the
|
||||||
|
// note on `postgres_indexes_for_relations_sql` above.
|
||||||
fn postgres_indexes_for_relations_compat_sql() -> &'static str {
|
fn postgres_indexes_for_relations_compat_sql() -> &'static str {
|
||||||
"SELECT t.oid::bigint AS relid, i.relname AS index_name, \
|
"SELECT t.oid::bigint AS relid, i.relname AS index_name, \
|
||||||
ARRAY( \
|
ARRAY( \
|
||||||
@@ -5101,6 +5126,9 @@ pub async fn list_schema_infos_with_system(pool: &Pool, show_system_schemas: boo
|
|||||||
.collect())
|
.collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Sibling of `postgres_columns_for_relations_sql`/`_compat_sql` above (~line
|
||||||
|
// 3086), for a single (schema, table) instead of a batch of oids — see the
|
||||||
|
// note there about why these aren't merged into a shared fragment.
|
||||||
const POSTGRES_COLUMNS_SQL: &str = "SELECT a.attname AS column_name, \
|
const POSTGRES_COLUMNS_SQL: &str = "SELECT a.attname AS column_name, \
|
||||||
format_type(a.atttypid, a.atttypmod) AS full_type, \
|
format_type(a.atttypid, a.atttypmod) AS full_type, \
|
||||||
COALESCE(c.is_nullable = 'YES', NOT a.attnotnull) AS is_nullable, \
|
COALESCE(c.is_nullable = 'YES', NOT a.attnotnull) AS is_nullable, \
|
||||||
@@ -5169,6 +5197,8 @@ const POSTGRES_COLUMNS_SQL: &str = "SELECT a.attname AS column_name, \
|
|||||||
AND a.attnum > 0 AND NOT a.attisdropped \
|
AND a.attnum > 0 AND NOT a.attisdropped \
|
||||||
ORDER BY a.attnum";
|
ORDER BY a.attnum";
|
||||||
|
|
||||||
|
// Compat-tier sibling of `postgres_columns_for_relations_compat_sql` (~line
|
||||||
|
// 3148) — see the note on `POSTGRES_COLUMNS_SQL` above.
|
||||||
const POSTGRES_COLUMNS_COMPAT_SQL: &str = "SELECT a.attname AS column_name, \
|
const POSTGRES_COLUMNS_COMPAT_SQL: &str = "SELECT a.attname AS column_name, \
|
||||||
format_type(a.atttypid, a.atttypmod) AS full_type, \
|
format_type(a.atttypid, a.atttypmod) AS full_type, \
|
||||||
COALESCE(c.is_nullable = 'YES', NOT a.attnotnull) AS is_nullable, \
|
COALESCE(c.is_nullable = 'YES', NOT a.attnotnull) AS is_nullable, \
|
||||||
@@ -5348,29 +5378,8 @@ async fn get_columns_with_sql(
|
|||||||
pub async fn get_columns(pool: &Pool, schema: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
|
pub async fn get_columns(pool: &Pool, schema: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
|
||||||
let schema = if schema.is_empty() { "public" } else { schema };
|
let schema = if schema.is_empty() { "public" } else { schema };
|
||||||
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
|
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
|
||||||
match get_columns_with_sql(&client, POSTGRES_COLUMNS_SQL, schema, table).await {
|
let tiers = [POSTGRES_COLUMNS_SQL, POSTGRES_COLUMNS_COMPAT_SQL, POSTGRES_COLUMNS_INFORMATION_SCHEMA_SQL];
|
||||||
Ok(columns) => Ok(columns),
|
query_with_compat_fallback("get_columns", &tiers, |sql| get_columns_with_sql(&client, sql, schema, table)).await
|
||||||
Err(primary_error) => match get_columns_with_sql(&client, POSTGRES_COLUMNS_COMPAT_SQL, schema, table).await {
|
|
||||||
Ok(columns) => Ok(columns),
|
|
||||||
Err(fallback_error) => {
|
|
||||||
let primary_message = pg_error_to_string(primary_error);
|
|
||||||
let fallback_message = pg_error_to_string(fallback_error);
|
|
||||||
match get_columns_with_sql(&client, POSTGRES_COLUMNS_INFORMATION_SCHEMA_SQL, schema, table).await {
|
|
||||||
Ok(columns) => Ok(columns),
|
|
||||||
Err(information_schema_error) => {
|
|
||||||
let information_schema_message = pg_error_to_string(information_schema_error);
|
|
||||||
log::debug!(
|
|
||||||
"[postgres][get_columns:compat-failed] primary_error={} fallback_error={} information_schema_error={}",
|
|
||||||
primary_message,
|
|
||||||
fallback_message,
|
|
||||||
information_schema_message
|
|
||||||
);
|
|
||||||
Err(information_schema_message)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn pg_quote_literal(value: &str) -> String {
|
fn pg_quote_literal(value: &str) -> String {
|
||||||
@@ -6372,6 +6381,8 @@ async fn execute_query_with_max_rows_inner(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Sibling of `postgres_indexes_for_relations_sql` (~line 3288), for a single
|
||||||
|
// (schema, table) instead of a batch of oids — see the note there.
|
||||||
const POSTGRES_INDEXES_SQL: &str = "SELECT i.relname AS index_name, \
|
const POSTGRES_INDEXES_SQL: &str = "SELECT i.relname AS index_name, \
|
||||||
array_agg(COALESCE(a.attname, pg_get_indexdef(ix.indexrelid, k.n::int, true)) ORDER BY k.n) AS columns, \
|
array_agg(COALESCE(a.attname, pg_get_indexdef(ix.indexrelid, k.n::int, true)) ORDER BY k.n) AS columns, \
|
||||||
ix.indisunique AS is_unique, \
|
ix.indisunique AS is_unique, \
|
||||||
@@ -6393,6 +6404,8 @@ const POSTGRES_INDEXES_SQL: &str = "SELECT i.relname AS index_name, \
|
|||||||
GROUP BY i.relname, i.oid, ix.indisunique, ix.indisprimary, ix.indpred, ix.indrelid, am.amname, ix.indnkeyatts, ix.indkey \
|
GROUP BY i.relname, i.oid, ix.indisunique, ix.indisprimary, ix.indpred, ix.indrelid, am.amname, ix.indnkeyatts, ix.indkey \
|
||||||
ORDER BY i.relname";
|
ORDER BY i.relname";
|
||||||
|
|
||||||
|
// Compat-tier sibling of `postgres_indexes_for_relations_compat_sql` (~line
|
||||||
|
// 3312) — see the note on `POSTGRES_INDEXES_SQL` above.
|
||||||
const POSTGRES_INDEXES_COMPAT_SQL: &str = "SELECT i.relname AS index_name, \
|
const POSTGRES_INDEXES_COMPAT_SQL: &str = "SELECT i.relname AS index_name, \
|
||||||
ARRAY( \
|
ARRAY( \
|
||||||
SELECT COALESCE(a.attname, pg_get_indexdef(ix.indexrelid, pos.n, true)) \
|
SELECT COALESCE(a.attname, pg_get_indexdef(ix.indexrelid, pos.n, true)) \
|
||||||
@@ -6465,7 +6478,7 @@ const POSTGRES_COLUMN_ACL_PRIVILEGES_SQL: &str =
|
|||||||
WHERE n.nspname = $1 AND c.relname = $2 AND c.relkind IN ('r', 'p') \
|
WHERE n.nspname = $1 AND c.relname = $2 AND c.relkind IN ('r', 'p') \
|
||||||
ORDER BY 5, 1, 2, 3, 4";
|
ORDER BY 5, 1, 2, 3, 4";
|
||||||
|
|
||||||
fn postgres_owner_object_type(relkind: &str) -> &str {
|
pub(crate) fn postgres_owner_object_type(relkind: &str) -> &str {
|
||||||
match relkind {
|
match relkind {
|
||||||
"r" => "TABLE",
|
"r" => "TABLE",
|
||||||
"v" => "VIEW",
|
"v" => "VIEW",
|
||||||
@@ -6478,6 +6491,29 @@ fn postgres_owner_object_type(relkind: &str) -> &str {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Looks up a relation's own `relkind` (e.g. `'r'`, `'v'`, `'S'`), regardless
|
||||||
|
/// of whether it's a kind this driver otherwise treats as a table. Used to
|
||||||
|
/// give a specific diagnostic ("it's a view, not a table") instead of a bare
|
||||||
|
/// "not found" when a `(schema, table)` request turns out not to be a
|
||||||
|
/// table/partition/foreign table.
|
||||||
|
pub(crate) async fn postgres_relation_relkind(
|
||||||
|
pool: &Pool,
|
||||||
|
schema: &str,
|
||||||
|
table: &str,
|
||||||
|
) -> Result<Option<String>, String> {
|
||||||
|
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
|
||||||
|
let row = client
|
||||||
|
.query_opt(
|
||||||
|
"SELECT c.relkind::text FROM pg_catalog.pg_class c \
|
||||||
|
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \
|
||||||
|
WHERE n.nspname = $1 AND c.relname = $2 LIMIT 1",
|
||||||
|
&[&schema, &table],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(pg_error_to_string)?;
|
||||||
|
row.map(|row| row.try_get::<_, String>(0)).transpose().map_err(pg_error_to_string)
|
||||||
|
}
|
||||||
|
|
||||||
async fn list_indexes_with_sql(
|
async fn list_indexes_with_sql(
|
||||||
client: &deadpool_postgres::Client,
|
client: &deadpool_postgres::Client,
|
||||||
sql: &str,
|
sql: &str,
|
||||||
@@ -6516,22 +6552,8 @@ async fn list_indexes_with_sql(
|
|||||||
|
|
||||||
pub async fn list_indexes(pool: &Pool, schema: &str, table: &str) -> Result<Vec<IndexInfo>, String> {
|
pub async fn list_indexes(pool: &Pool, schema: &str, table: &str) -> Result<Vec<IndexInfo>, String> {
|
||||||
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
|
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
|
||||||
match list_indexes_with_sql(&client, POSTGRES_INDEXES_SQL, schema, table).await {
|
let tiers = [POSTGRES_INDEXES_SQL, POSTGRES_INDEXES_COMPAT_SQL];
|
||||||
Ok(indexes) => Ok(indexes),
|
query_with_compat_fallback("list_indexes", &tiers, |sql| list_indexes_with_sql(&client, sql, schema, table)).await
|
||||||
Err(primary_error) => match list_indexes_with_sql(&client, POSTGRES_INDEXES_COMPAT_SQL, schema, table).await {
|
|
||||||
Ok(indexes) => Ok(indexes),
|
|
||||||
Err(fallback_error) => {
|
|
||||||
let primary_message = pg_error_to_string(primary_error);
|
|
||||||
let fallback_message = pg_error_to_string(fallback_error);
|
|
||||||
log::debug!(
|
|
||||||
"[postgres][list_indexes:compat-failed] primary_error={} fallback_error={}",
|
|
||||||
primary_message,
|
|
||||||
fallback_message
|
|
||||||
);
|
|
||||||
Err(fallback_message)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Names of same-table indexes whose `pg_index.indisvalid` is `false`.
|
/// Names of same-table indexes whose `pg_index.indisvalid` is `false`.
|
||||||
|
|||||||
@@ -9812,9 +9812,17 @@ pub async fn pg_ddl_with_partitions(
|
|||||||
let Some(root) =
|
let Some(root) =
|
||||||
tree.iter().find(|node| !node.parent_oid.is_some_and(|parent_oid| tree_oids.contains(&parent_oid)))
|
tree.iter().find(|node| !node.parent_oid.is_some_and(|parent_oid| tree_oids.contains(&parent_oid)))
|
||||||
else {
|
else {
|
||||||
return Err(format!(
|
// The tree query only matches relkind IN ('r','p','f'), so an empty
|
||||||
"relation \"{schema}\".\"{table}\" was not found or is not a table/partition/foreign table"
|
// tree could mean the relation doesn't exist at all, or that it
|
||||||
));
|
// exists but is a view/sequence/other non-table object. Look up its
|
||||||
|
// relkind (if any) to tell those two cases apart in the error.
|
||||||
|
return Err(match db::postgres::postgres_relation_relkind(pool, schema, table).await {
|
||||||
|
Ok(Some(relkind)) => format!(
|
||||||
|
"relation \"{schema}\".\"{table}\" is a {}, not a table/partition/foreign table",
|
||||||
|
db::postgres::postgres_owner_object_type(&relkind)
|
||||||
|
),
|
||||||
|
_ => format!("relation \"{schema}\".\"{table}\" was not found or is not a table/partition/foreign table"),
|
||||||
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
let oids: Vec<i64> = tree.iter().map(|node| node.oid).collect();
|
let oids: Vec<i64> = tree.iter().map(|node| node.oid).collect();
|
||||||
@@ -9841,6 +9849,18 @@ pub async fn pg_ddl_with_partitions(
|
|||||||
db::postgres::get_table_partition_local_objects_for_relations(pool, &oids),
|
db::postgres::get_table_partition_local_objects_for_relations(pool, &oids),
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
// `list_foreign_keys_for_relations` is keyed by (schema, table) since its
|
||||||
|
// query is information_schema-based, unlike every other *_by_oid map
|
||||||
|
// here. Remap it once so the render loop below can look fkeys up by oid
|
||||||
|
// like everything else, instead of cloning a fresh (schema, table) key
|
||||||
|
// on every node it renders.
|
||||||
|
let relation_oid_by_pair: HashMap<(String, String), i64> =
|
||||||
|
relations.into_iter().map(|(oid, schema, table)| ((schema, table), oid)).collect();
|
||||||
|
let fkeys_by_oid: HashMap<i64, Vec<db::ForeignKeyInfo>> = fkeys_by_relation
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|(key, fkeys)| relation_oid_by_pair.get(&key).map(|oid| (*oid, fkeys)))
|
||||||
|
.collect();
|
||||||
|
|
||||||
// Group children by parent oid, each group ordered by relname to match
|
// Group children by parent oid, each group ordered by relname to match
|
||||||
// `list_table_partitions`' `ORDER BY c.relname` (today's traversal order).
|
// `list_table_partitions`' `ORDER BY c.relname` (today's traversal order).
|
||||||
let mut children_by_parent: HashMap<i64, Vec<&db::postgres::PostgresPartitionTreeNode>> = HashMap::new();
|
let mut children_by_parent: HashMap<i64, Vec<&db::postgres::PostgresPartitionTreeNode>> = HashMap::new();
|
||||||
@@ -9859,7 +9879,7 @@ pub async fn pg_ddl_with_partitions(
|
|||||||
&children_by_parent,
|
&children_by_parent,
|
||||||
&columns_by_oid,
|
&columns_by_oid,
|
||||||
&indexes_by_oid,
|
&indexes_by_oid,
|
||||||
&fkeys_by_relation,
|
&fkeys_by_oid,
|
||||||
&comments_by_oid,
|
&comments_by_oid,
|
||||||
&triggers_by_oid,
|
&triggers_by_oid,
|
||||||
&checks_by_oid,
|
&checks_by_oid,
|
||||||
@@ -9880,7 +9900,7 @@ fn render_postgres_partition_tree_node(
|
|||||||
children_by_parent: &HashMap<i64, Vec<&db::postgres::PostgresPartitionTreeNode>>,
|
children_by_parent: &HashMap<i64, Vec<&db::postgres::PostgresPartitionTreeNode>>,
|
||||||
columns_by_oid: &HashMap<i64, Vec<db::ColumnInfo>>,
|
columns_by_oid: &HashMap<i64, Vec<db::ColumnInfo>>,
|
||||||
indexes_by_oid: &HashMap<i64, Vec<db::IndexInfo>>,
|
indexes_by_oid: &HashMap<i64, Vec<db::IndexInfo>>,
|
||||||
fkeys_by_relation: &HashMap<(String, String), Vec<db::ForeignKeyInfo>>,
|
fkeys_by_oid: &HashMap<i64, Vec<db::ForeignKeyInfo>>,
|
||||||
comments_by_oid: &HashMap<i64, Option<String>>,
|
comments_by_oid: &HashMap<i64, Option<String>>,
|
||||||
triggers_by_oid: &HashMap<i64, Vec<String>>,
|
triggers_by_oid: &HashMap<i64, Vec<String>>,
|
||||||
checks_by_oid: &HashMap<i64, Vec<(String, String)>>,
|
checks_by_oid: &HashMap<i64, Vec<(String, String)>>,
|
||||||
@@ -9903,8 +9923,8 @@ fn render_postgres_partition_tree_node(
|
|||||||
|
|
||||||
let columns = columns_by_oid.get(&node.oid).unwrap_or(&empty_columns);
|
let columns = columns_by_oid.get(&node.oid).unwrap_or(&empty_columns);
|
||||||
let indexes = indexes_by_oid.get(&node.oid).unwrap_or(&empty_indexes);
|
let indexes = indexes_by_oid.get(&node.oid).unwrap_or(&empty_indexes);
|
||||||
let fkeys = fkeys_by_relation.get(&(node.schema.clone(), node.table.clone())).unwrap_or(&empty_fkeys);
|
let fkeys = fkeys_by_oid.get(&node.oid).unwrap_or(&empty_fkeys);
|
||||||
let comment = comments_by_oid.get(&node.oid).cloned().flatten();
|
let comment = comments_by_oid.get(&node.oid).and_then(|comment| comment.as_deref());
|
||||||
let triggers = triggers_by_oid.get(&node.oid).unwrap_or(&empty_triggers);
|
let triggers = triggers_by_oid.get(&node.oid).unwrap_or(&empty_triggers);
|
||||||
let checks = checks_by_oid.get(&node.oid).unwrap_or(&empty_checks);
|
let checks = checks_by_oid.get(&node.oid).unwrap_or(&empty_checks);
|
||||||
let local_objects = local_objects_by_oid.get(&node.oid).unwrap_or(&empty_local_objects);
|
let local_objects = local_objects_by_oid.get(&node.oid).unwrap_or(&empty_local_objects);
|
||||||
@@ -9920,7 +9940,7 @@ fn render_postgres_partition_tree_node(
|
|||||||
indexes,
|
indexes,
|
||||||
fkeys,
|
fkeys,
|
||||||
checks,
|
checks,
|
||||||
comment.as_deref(),
|
comment,
|
||||||
&node.partition_info,
|
&node.partition_info,
|
||||||
local_objects,
|
local_objects,
|
||||||
),
|
),
|
||||||
@@ -10195,13 +10215,82 @@ fn append_opengauss_trigger_definitions(mut ddl: String, trigger_definitions: &[
|
|||||||
ddl
|
ddl
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Cloudberry's native `pg_get_tabledef` (a community-maintained PL/pgSQL
|
||||||
|
/// function most Cloudberry/Greenplum installs have, not a built-in) renders
|
||||||
|
/// exactly one relation — for a partitioned table it emits that table's own
|
||||||
|
/// `CREATE TABLE ... PARTITION BY ...`, never its partitions' own `CREATE
|
||||||
|
/// TABLE ... PARTITION OF ...` statements, regardless of whether the table
|
||||||
|
/// was created with classic Greenplum or PostgreSQL-style declarative
|
||||||
|
/// partition syntax. When `include_partitions` is requested, fetch the same
|
||||||
|
/// partition tree the plain-Postgres path uses and append each descendant's
|
||||||
|
/// own native DDL, so Cloudberry's more accurate native rendering (storage
|
||||||
|
/// options, distribution policy, external-table clauses) is still used per
|
||||||
|
/// relation instead of falling back to the generic renderer for the whole
|
||||||
|
/// tree.
|
||||||
|
async fn cloudberry_native_tree_ddl(
|
||||||
|
pool: &deadpool_postgres::Pool,
|
||||||
|
schema: &str,
|
||||||
|
table: &str,
|
||||||
|
include_partitions: bool,
|
||||||
|
) -> Result<String, String> {
|
||||||
|
let root_ddl = db::cloudberry::table_ddl(pool, schema, table).await?;
|
||||||
|
if !include_partitions {
|
||||||
|
return Ok(root_ddl);
|
||||||
|
}
|
||||||
|
let tree = db::postgres::fetch_postgres_partition_tree(pool, schema, table).await?;
|
||||||
|
if tree.len() <= 1 {
|
||||||
|
return Ok(root_ddl);
|
||||||
|
}
|
||||||
|
let tree_oids: HashSet<i64> = tree.iter().map(|node| node.oid).collect();
|
||||||
|
let Some(root) =
|
||||||
|
tree.iter().find(|node| !node.parent_oid.is_some_and(|parent_oid| tree_oids.contains(&parent_oid)))
|
||||||
|
else {
|
||||||
|
return Ok(root_ddl);
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut children_by_parent: HashMap<i64, Vec<&db::postgres::PostgresPartitionTreeNode>> = HashMap::new();
|
||||||
|
for node in &tree {
|
||||||
|
if let Some(parent_oid) = node.parent_oid {
|
||||||
|
children_by_parent.entry(parent_oid).or_default().push(node);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for children in children_by_parent.values_mut() {
|
||||||
|
children.sort_by(|a, b| a.table.cmp(&b.table));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut descendants = Vec::new();
|
||||||
|
let mut visited: HashSet<i64> = HashSet::new();
|
||||||
|
let mut stack: Vec<&db::postgres::PostgresPartitionTreeNode> = vec![root];
|
||||||
|
while let Some(node) = stack.pop() {
|
||||||
|
if !visited.insert(node.oid) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if node.oid != root.oid {
|
||||||
|
descendants.push(node);
|
||||||
|
}
|
||||||
|
if let Some(children) = children_by_parent.get(&node.oid) {
|
||||||
|
for child in children.iter().rev() {
|
||||||
|
stack.push(child);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut ddl = root_ddl;
|
||||||
|
for node in descendants {
|
||||||
|
let child_ddl = db::cloudberry::table_ddl(pool, &node.schema, &node.table).await?;
|
||||||
|
ddl.push('\n');
|
||||||
|
ddl.push_str(&child_ddl);
|
||||||
|
}
|
||||||
|
Ok(ddl)
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn cloudberry_ddl(
|
pub async fn cloudberry_ddl(
|
||||||
pool: &deadpool_postgres::Pool,
|
pool: &deadpool_postgres::Pool,
|
||||||
schema: &str,
|
schema: &str,
|
||||||
table: &str,
|
table: &str,
|
||||||
include_partitions: bool,
|
include_partitions: bool,
|
||||||
) -> Result<String, String> {
|
) -> Result<String, String> {
|
||||||
match db::cloudberry::table_ddl(pool, schema, table).await {
|
match cloudberry_native_tree_ddl(pool, schema, table, include_partitions).await {
|
||||||
Ok(ddl) => Ok(ddl),
|
Ok(ddl) => Ok(ddl),
|
||||||
Err(native_error) => {
|
Err(native_error) => {
|
||||||
let base_ddl = pg_ddl_for_options(pool, schema, table, include_partitions).await.map_err(|fallback_error| {
|
let base_ddl = pg_ddl_for_options(pool, schema, table, include_partitions).await.map_err(|fallback_error| {
|
||||||
@@ -10228,6 +10317,39 @@ pub async fn opentenbase_ddl(
|
|||||||
include_partitions: bool,
|
include_partitions: bool,
|
||||||
) -> Result<String, String> {
|
) -> Result<String, String> {
|
||||||
let ddl = pg_ddl_for_options(pool, schema, table, include_partitions).await?;
|
let ddl = pg_ddl_for_options(pool, schema, table, include_partitions).await?;
|
||||||
|
if include_partitions {
|
||||||
|
// The rendered ddl may cover the whole partition tree (root plus
|
||||||
|
// every partition), each as its own `CREATE [FOREIGN] TABLE`
|
||||||
|
// statement, so distribution policies need to be looked up and
|
||||||
|
// applied per relation rather than once for the root.
|
||||||
|
let relations: Vec<(String, String)> = db::ddl_scan::top_level_statement_ranges(&ddl)
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|range| db::ddl_scan::parse_create_table_relation(&ddl[range]))
|
||||||
|
.collect();
|
||||||
|
return match db::opentenbase::table_distribution_for_relations(pool, &relations).await {
|
||||||
|
Ok(distributions) => match db::opentenbase::append_distribution_clauses(&ddl, &distributions) {
|
||||||
|
Ok(ddl) => Ok(ddl),
|
||||||
|
Err(error) => {
|
||||||
|
log::warn!(
|
||||||
|
"[schema][opentenbase:table-ddl-distribution-render-fallback] schema={} table={} error={}",
|
||||||
|
schema,
|
||||||
|
table,
|
||||||
|
error
|
||||||
|
);
|
||||||
|
Ok(ddl)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Err(error) => {
|
||||||
|
log::warn!(
|
||||||
|
"[schema][opentenbase:table-ddl-distribution-query-fallback] schema={} table={} error={}",
|
||||||
|
schema,
|
||||||
|
table,
|
||||||
|
error
|
||||||
|
);
|
||||||
|
Ok(ddl)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
match db::opentenbase::table_distribution(pool, schema, table).await {
|
match db::opentenbase::table_distribution(pool, schema, table).await {
|
||||||
Ok(Some(distribution)) => match db::opentenbase::append_distribution_clause(&ddl, &distribution) {
|
Ok(Some(distribution)) => match db::opentenbase::append_distribution_clause(&ddl, &distribution) {
|
||||||
Ok(ddl) => Ok(ddl),
|
Ok(ddl) => Ok(ddl),
|
||||||
|
|||||||
@@ -10,164 +10,17 @@
|
|||||||
//! number of partitions.
|
//! number of partitions.
|
||||||
//! Needs local Docker; skips cleanly when unavailable.
|
//! Needs local Docker; skips cleanly when unavailable.
|
||||||
|
|
||||||
|
mod support;
|
||||||
|
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
|
|
||||||
use dbx_core::connection::AppState;
|
use dbx_core::connection::AppState;
|
||||||
use dbx_core::database_export::{export_database_sql_core, DatabaseExportRequest};
|
use dbx_core::database_export::{export_database_sql_core, DatabaseExportRequest};
|
||||||
use dbx_core::models::connection::{ConnectionConfig, DatabaseType};
|
|
||||||
use dbx_core::storage::Storage;
|
use dbx_core::storage::Storage;
|
||||||
|
use support::{
|
||||||
struct DockerPostgres {
|
postgres_test_config, psql, psql_allow_failure, start_docker_postgres, start_docker_postgres_with_server_args,
|
||||||
name: String,
|
DockerPostgres,
|
||||||
port: u16,
|
};
|
||||||
remove_on_drop: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for DockerPostgres {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
if self.remove_on_drop {
|
|
||||||
let _ = Command::new("docker").args(["rm", "-f", &self.name]).status();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn docker_ready() -> bool {
|
|
||||||
Command::new("docker")
|
|
||||||
.args(["version", "--format", "{{.Server.Version}}"])
|
|
||||||
.output()
|
|
||||||
.map(|output| output.status.success())
|
|
||||||
.unwrap_or(false)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn start_docker_postgres(name_prefix: &str) -> Option<DockerPostgres> {
|
|
||||||
if let Ok(name) = std::env::var("DBX_TEST_POSTGRES_CONTAINER") {
|
|
||||||
let port = std::env::var("DBX_TEST_POSTGRES_PORT")
|
|
||||||
.expect("DBX_TEST_POSTGRES_PORT must accompany DBX_TEST_POSTGRES_CONTAINER")
|
|
||||||
.parse()
|
|
||||||
.expect("DBX_TEST_POSTGRES_PORT must be a valid port");
|
|
||||||
return Some(DockerPostgres { name, port, remove_on_drop: false });
|
|
||||||
}
|
|
||||||
if !docker_ready() {
|
|
||||||
eprintln!("skipping docker-backed partition ddl test because Docker is unavailable");
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
|
|
||||||
let port = portpicker::pick_unused_port().expect("pick unused postgres port");
|
|
||||||
let container =
|
|
||||||
DockerPostgres { name: format!("{name_prefix}-{}", uuid::Uuid::new_v4()), port, remove_on_drop: true };
|
|
||||||
|
|
||||||
let status = Command::new("docker")
|
|
||||||
.args([
|
|
||||||
"run",
|
|
||||||
"-d",
|
|
||||||
"--rm",
|
|
||||||
"--name",
|
|
||||||
&container.name,
|
|
||||||
"-e",
|
|
||||||
"POSTGRES_PASSWORD=postgres",
|
|
||||||
"-e",
|
|
||||||
"POSTGRES_USER=postgres",
|
|
||||||
"-e",
|
|
||||||
"POSTGRES_DB=postgres",
|
|
||||||
"-p",
|
|
||||||
&format!("{port}:5432"),
|
|
||||||
"postgres:16-alpine",
|
|
||||||
])
|
|
||||||
.status()
|
|
||||||
.expect("start docker postgres");
|
|
||||||
assert!(status.success(), "docker run postgres container should succeed");
|
|
||||||
|
|
||||||
let mut consecutive_ok = 0;
|
|
||||||
for _ in 0..120 {
|
|
||||||
let ready = Command::new("docker")
|
|
||||||
.args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-c", "SELECT 1"])
|
|
||||||
.output()
|
|
||||||
.map(|output| output.status.success())
|
|
||||||
.unwrap_or(false);
|
|
||||||
consecutive_ok = if ready { consecutive_ok + 1 } else { 0 };
|
|
||||||
if consecutive_ok >= 2 {
|
|
||||||
return Some(container);
|
|
||||||
}
|
|
||||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
|
||||||
}
|
|
||||||
panic!("postgres container did not become ready");
|
|
||||||
}
|
|
||||||
|
|
||||||
fn psql(container: &DockerPostgres, sql: &str) {
|
|
||||||
let output = Command::new("docker")
|
|
||||||
.args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-c", sql])
|
|
||||||
.output()
|
|
||||||
.expect("run psql");
|
|
||||||
assert!(output.status.success(), "psql failed: {}", String::from_utf8_lossy(&output.stderr));
|
|
||||||
}
|
|
||||||
|
|
||||||
fn psql_allow_failure(container: &DockerPostgres, sql: &str) -> bool {
|
|
||||||
Command::new("docker")
|
|
||||||
.args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-c", sql])
|
|
||||||
.output()
|
|
||||||
.map(|output| output.status.success())
|
|
||||||
.unwrap_or(false)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn postgres_test_config(id: &str, port: u16) -> ConnectionConfig {
|
|
||||||
ConnectionConfig {
|
|
||||||
docs_notes_path: None,
|
|
||||||
id: id.to_string(),
|
|
||||||
name: id.to_string(),
|
|
||||||
note: String::new(),
|
|
||||||
db_type: DatabaseType::Postgres,
|
|
||||||
driver_profile: None,
|
|
||||||
driver_label: None,
|
|
||||||
url_params: None,
|
|
||||||
agent_java_options: Vec::new(),
|
|
||||||
host: "127.0.0.1".to_string(),
|
|
||||||
port,
|
|
||||||
username: "postgres".to_string(),
|
|
||||||
password: "postgres".to_string(),
|
|
||||||
database: Some("postgres".to_string()),
|
|
||||||
default_schema: None,
|
|
||||||
visible_databases: None,
|
|
||||||
visible_schemas: None,
|
|
||||||
attached_databases: Vec::new(),
|
|
||||||
init_script: None,
|
|
||||||
color: None,
|
|
||||||
transport_layers: Vec::new(),
|
|
||||||
connect_timeout_secs: 5,
|
|
||||||
query_timeout_secs: 30,
|
|
||||||
idle_timeout_secs: 60,
|
|
||||||
keepalive_interval_secs: 0,
|
|
||||||
ssl: false,
|
|
||||||
ca_cert_path: String::new(),
|
|
||||||
client_cert_path: String::new(),
|
|
||||||
client_key_path: String::new(),
|
|
||||||
sysdba: false,
|
|
||||||
oracle_connection_type: None,
|
|
||||||
connection_string: None,
|
|
||||||
redis_connection_mode: None,
|
|
||||||
redis_sentinel_master: String::new(),
|
|
||||||
redis_sentinel_nodes: String::new(),
|
|
||||||
redis_sentinel_username: String::new(),
|
|
||||||
redis_sentinel_password: String::new(),
|
|
||||||
redis_sentinel_tls: false,
|
|
||||||
redis_cluster_nodes: String::new(),
|
|
||||||
redis_key_separator: dbx_core::models::connection::default_redis_key_separator(),
|
|
||||||
redis_scan_page_size: None,
|
|
||||||
redis_database_aliases: Default::default(),
|
|
||||||
etcd_endpoints: String::new(),
|
|
||||||
gbase_server: String::new(),
|
|
||||||
informix_server: String::new(),
|
|
||||||
external_config: None,
|
|
||||||
jdbc_driver_class: None,
|
|
||||||
jdbc_driver_paths: Vec::new(),
|
|
||||||
one_time: false,
|
|
||||||
save_password: true,
|
|
||||||
read_only: false,
|
|
||||||
is_production: false,
|
|
||||||
production_databases: vec![],
|
|
||||||
show_system_schemas: false,
|
|
||||||
database_info: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Full database export of a partitioned schema must not duplicate any
|
/// Full database export of a partitioned schema must not duplicate any
|
||||||
/// relation's `CREATE TABLE`/`CREATE FOREIGN TABLE`, must faithfully replay
|
/// relation's `CREATE TABLE`/`CREATE FOREIGN TABLE`, must faithfully replay
|
||||||
@@ -433,3 +286,156 @@ async fn partition_tree_ddl_query_count_does_not_scale_with_partition_count() {
|
|||||||
"query count should not scale with partition count: small={small_count} large={large_count}"
|
"query count should not scale with partition count: small={small_count} large={large_count}"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A tree-DDL request for a relation that exists but isn't a
|
||||||
|
/// table/partition/foreign table (e.g. a view) must say so specifically,
|
||||||
|
/// not fall back to a generic "not found" message that also covers a
|
||||||
|
/// relation that doesn't exist at all.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn display_ddl_error_distinguishes_wrong_relkind_from_missing_relation() {
|
||||||
|
let Some(container) = start_docker_postgres("dbx-partition-ddl-relkind-error") else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
|
||||||
|
psql(&container, "CREATE VIEW v AS SELECT 1 AS id");
|
||||||
|
|
||||||
|
let dir = std::env::temp_dir().join(format!("dbx-partition-relkind-error-{}", uuid::Uuid::new_v4()));
|
||||||
|
std::fs::create_dir_all(&dir).unwrap();
|
||||||
|
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||||
|
let state = AppState::new(storage);
|
||||||
|
let connection_id = "partition-relkind-error-conn";
|
||||||
|
state.configs.write().await.insert(connection_id.to_string(), postgres_test_config(connection_id, container.port));
|
||||||
|
|
||||||
|
let view_error =
|
||||||
|
dbx_core::schema::get_table_display_ddl_core(&state, connection_id, "postgres", "public", "v", None)
|
||||||
|
.await
|
||||||
|
.expect_err("requesting tree DDL for a view must fail");
|
||||||
|
assert!(view_error.contains("is a VIEW"), "expected a relkind-specific error, got: {view_error}");
|
||||||
|
|
||||||
|
let missing_error = dbx_core::schema::get_table_display_ddl_core(
|
||||||
|
&state,
|
||||||
|
connection_id,
|
||||||
|
"postgres",
|
||||||
|
"public",
|
||||||
|
"does_not_exist",
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("requesting tree DDL for a missing relation must fail");
|
||||||
|
assert!(missing_error.contains("was not found"), "expected a not-found error, got: {missing_error}");
|
||||||
|
assert!(!missing_error.contains("is a"), "not-found error should not claim a relkind: {missing_error}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Cloudberry/Greenplum's `pg_get_tabledef` (a community PL/pgSQL function
|
||||||
|
/// most installs have, not a catalog builtin) renders exactly the one
|
||||||
|
/// relation it's asked for — never its partitions' own `CREATE TABLE ...
|
||||||
|
/// PARTITION OF ...` statements. `cloudberry_ddl(..., include_partitions:
|
||||||
|
/// true)` must still return every partition's DDL by walking the same
|
||||||
|
/// partition tree the plain-Postgres path uses, instead of silently
|
||||||
|
/// returning just the root's DDL whenever the native call succeeds. This
|
||||||
|
/// installs a minimal stand-in `pg_get_tabledef` (mirroring the real one's
|
||||||
|
/// single-relation behavior) rather than the full upstream script, since
|
||||||
|
/// only that single-relation behavior is what the fix under test reacts to.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cloudberry_native_ddl_still_includes_every_partition() {
|
||||||
|
let Some(container) = start_docker_postgres("dbx-cloudberry-partition-ddl") else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
|
||||||
|
psql(
|
||||||
|
&container,
|
||||||
|
"CREATE OR REPLACE FUNCTION pg_get_tabledef(in_schema text, in_table text, _verbose boolean) \
|
||||||
|
RETURNS text AS $$ SELECT format('CREATE TABLE \"%s\".\"%s\" (id integer) /* stub-native-ddl */;', in_schema, in_table) $$ \
|
||||||
|
LANGUAGE sql",
|
||||||
|
);
|
||||||
|
psql(&container, "CREATE TABLE t (id integer NOT NULL) PARTITION BY RANGE (id)");
|
||||||
|
psql(&container, "CREATE TABLE t_p1 PARTITION OF t FOR VALUES FROM (0) TO (100)");
|
||||||
|
psql(&container, "CREATE TABLE t_p2 PARTITION OF t FOR VALUES FROM (100) TO (200)");
|
||||||
|
|
||||||
|
let url = format!("postgres://postgres:postgres@127.0.0.1:{}/postgres", container.port);
|
||||||
|
let pool = dbx_core::db::postgres::connect(&url, std::time::Duration::from_secs(5)).await.expect("connect");
|
||||||
|
|
||||||
|
let ddl =
|
||||||
|
dbx_core::schema::cloudberry_ddl(&pool, "public", "t", true).await.expect("cloudberry ddl with partitions");
|
||||||
|
assert_eq!(
|
||||||
|
ddl.matches("stub-native-ddl").count(),
|
||||||
|
3,
|
||||||
|
"expected the root plus both partitions to each get their own native DDL: {ddl}"
|
||||||
|
);
|
||||||
|
assert!(ddl.contains("CREATE TABLE \"public\".\"t\""));
|
||||||
|
assert!(ddl.contains("CREATE TABLE \"public\".\"t_p1\""));
|
||||||
|
assert!(ddl.contains("CREATE TABLE \"public\".\"t_p2\""));
|
||||||
|
|
||||||
|
let single_ddl =
|
||||||
|
dbx_core::schema::cloudberry_ddl(&pool, "public", "t", false).await.expect("cloudberry ddl without partitions");
|
||||||
|
assert_eq!(single_ddl.matches("stub-native-ddl").count(), 1, "expected only the root's own DDL: {single_ddl}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `opentenbase_ddl(..., include_partitions: true)` must query and append
|
||||||
|
/// each partition's own `DISTRIBUTE BY` clause, not just the tree root's.
|
||||||
|
/// OpenTenBase itself has no simple single-container way to stand up (its
|
||||||
|
/// own docs only cover a multi-node `pgxc_ctl deploy`/`init` cluster built
|
||||||
|
/// from source), so this runs the actual, unmodified
|
||||||
|
/// `TABLE_DISTRIBUTION_FOR_RELATIONS_SQL` query against a real PostgreSQL
|
||||||
|
/// container with a stand-in `pg_catalog.pgxc_class` table shaped like
|
||||||
|
/// OpenTenBase's real one — exercising the real SQL (join, `unnest($1::
|
||||||
|
/// text[], $2::text[])` parameter binding, casts, row parsing), not just the
|
||||||
|
/// clause-insertion string logic around it.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn opentenbase_tree_ddl_gives_every_partition_its_own_distribute_by() {
|
||||||
|
let Some(container) =
|
||||||
|
start_docker_postgres_with_server_args("dbx-opentenbase-partition-ddl", &["-c", "allow_system_table_mods=on"])
|
||||||
|
else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
|
||||||
|
psql(
|
||||||
|
&container,
|
||||||
|
"CREATE TABLE pg_catalog.pgxc_class (pcrelid oid PRIMARY KEY, pclocatortype \"char\", discolnums smallint[])",
|
||||||
|
);
|
||||||
|
psql(&container, "CREATE TABLE t (id integer NOT NULL, region text NOT NULL) PARTITION BY RANGE (id)");
|
||||||
|
psql(&container, "CREATE TABLE t_p1 PARTITION OF t FOR VALUES FROM (0) TO (100)");
|
||||||
|
psql(&container, "CREATE TABLE t_p2 PARTITION OF t FOR VALUES FROM (100) TO (200)");
|
||||||
|
// Root distributes by `region` (attnum 2); t_p1 distributes by `id`
|
||||||
|
// (attnum 1) instead, to prove each relation's own policy is looked up
|
||||||
|
// independently rather than the root's being reused for every relation.
|
||||||
|
// t_p2 deliberately has no pgxc_class row at all — it must be left
|
||||||
|
// without a DISTRIBUTE BY clause rather than erroring the whole DDL.
|
||||||
|
psql(&container, "INSERT INTO pg_catalog.pgxc_class VALUES ('t'::regclass, 'H', ARRAY[2]::smallint[])");
|
||||||
|
psql(&container, "INSERT INTO pg_catalog.pgxc_class VALUES ('t_p1'::regclass, 'H', ARRAY[1]::smallint[])");
|
||||||
|
|
||||||
|
let url = format!("postgres://postgres:postgres@127.0.0.1:{}/postgres", container.port);
|
||||||
|
let pool = dbx_core::db::postgres::connect(&url, std::time::Duration::from_secs(5)).await.expect("connect");
|
||||||
|
|
||||||
|
let distributions = dbx_core::db::opentenbase::table_distribution_for_relations(
|
||||||
|
&pool,
|
||||||
|
&[("public".to_string(), "t".to_string()), ("public".to_string(), "t_p1".to_string())],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("batched distribution query");
|
||||||
|
assert_eq!(
|
||||||
|
distributions.get(&("public".to_string(), "t".to_string())),
|
||||||
|
Some(&dbx_core::db::opentenbase::DistributionPolicy::Hash(vec!["region".to_string()]))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
distributions.get(&("public".to_string(), "t_p1".to_string())),
|
||||||
|
Some(&dbx_core::db::opentenbase::DistributionPolicy::Hash(vec!["id".to_string()]))
|
||||||
|
);
|
||||||
|
|
||||||
|
let ddl = dbx_core::schema::opentenbase_ddl(&pool, "public", "t", true).await.expect("opentenbase tree ddl");
|
||||||
|
assert!(
|
||||||
|
ddl.contains("\"t\" (\n \"id\" integer NOT NULL,\n \"region\" text NOT NULL\n) PARTITION BY RANGE (id)\nDISTRIBUTE BY HASH (\"region\");"),
|
||||||
|
"root should get its own DISTRIBUTE BY HASH (region): {ddl}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
ddl.contains(
|
||||||
|
"\"t_p1\" PARTITION OF \"public\".\"t\" FOR VALUES FROM (0) TO (100)\nDISTRIBUTE BY HASH (\"id\");"
|
||||||
|
),
|
||||||
|
"t_p1 should get its own DISTRIBUTE BY HASH (id), independent of the root's: {ddl}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
ddl.contains("\"t_p2\" PARTITION OF \"public\".\"t\" FOR VALUES FROM (100) TO (200);"),
|
||||||
|
"t_p2 has no pgxc_class row and must not get a DISTRIBUTE BY clause: {ddl}"
|
||||||
|
);
|
||||||
|
assert_eq!(ddl.matches("DISTRIBUTE BY").count(), 2);
|
||||||
|
}
|
||||||
@@ -1,147 +1,12 @@
|
|||||||
//! 整库 SQL 导出的端到端回归测试(元数据并发预取后输出必须与逐表查询一致)。
|
//! 整库 SQL 导出的端到端回归测试(元数据并发预取后输出必须与逐表查询一致)。
|
||||||
//! 需要本机 Docker;不可用时自动跳过。
|
//! 需要本机 Docker;不可用时自动跳过。
|
||||||
|
|
||||||
use std::process::Command;
|
mod support;
|
||||||
|
|
||||||
use dbx_core::connection::AppState;
|
use dbx_core::connection::AppState;
|
||||||
use dbx_core::database_export::{export_database_sql_core, DatabaseExportRequest};
|
use dbx_core::database_export::{export_database_sql_core, DatabaseExportRequest};
|
||||||
use dbx_core::models::connection::{ConnectionConfig, DatabaseType};
|
|
||||||
use dbx_core::storage::Storage;
|
use dbx_core::storage::Storage;
|
||||||
|
use support::{postgres_test_config, psql, start_docker_postgres};
|
||||||
struct DockerPostgres {
|
|
||||||
name: String,
|
|
||||||
port: u16,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for DockerPostgres {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
let _ = Command::new("docker").args(["rm", "-f", &self.name]).status();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn docker_ready() -> bool {
|
|
||||||
Command::new("docker")
|
|
||||||
.args(["version", "--format", "{{.Server.Version}}"])
|
|
||||||
.output()
|
|
||||||
.map(|output| output.status.success())
|
|
||||||
.unwrap_or(false)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn start_docker_postgres() -> Option<DockerPostgres> {
|
|
||||||
if !docker_ready() {
|
|
||||||
eprintln!("skipping docker-backed export test because Docker is unavailable");
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
|
|
||||||
let port = portpicker::pick_unused_port().expect("pick unused postgres port");
|
|
||||||
let container = DockerPostgres { name: format!("dbx-export-prefetch-{}", uuid::Uuid::new_v4()), port };
|
|
||||||
|
|
||||||
let status = Command::new("docker")
|
|
||||||
.args([
|
|
||||||
"run",
|
|
||||||
"-d",
|
|
||||||
"--rm",
|
|
||||||
"--name",
|
|
||||||
&container.name,
|
|
||||||
"-e",
|
|
||||||
"POSTGRES_PASSWORD=postgres",
|
|
||||||
"-e",
|
|
||||||
"POSTGRES_USER=postgres",
|
|
||||||
"-e",
|
|
||||||
"POSTGRES_DB=postgres",
|
|
||||||
"-p",
|
|
||||||
&format!("{port}:5432"),
|
|
||||||
"postgres:16-alpine",
|
|
||||||
])
|
|
||||||
.status()
|
|
||||||
.expect("start docker postgres");
|
|
||||||
assert!(status.success(), "docker run postgres container should succeed");
|
|
||||||
|
|
||||||
// postgres 镜像 initdb 期间会短暂拉起再重启服务,单次探测可能命中引导期,
|
|
||||||
// 要求 SELECT 1 连续两次成功才算就绪
|
|
||||||
let mut consecutive_ok = 0;
|
|
||||||
for _ in 0..120 {
|
|
||||||
let ready = Command::new("docker")
|
|
||||||
.args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-c", "SELECT 1"])
|
|
||||||
.output()
|
|
||||||
.map(|output| output.status.success())
|
|
||||||
.unwrap_or(false);
|
|
||||||
consecutive_ok = if ready { consecutive_ok + 1 } else { 0 };
|
|
||||||
if consecutive_ok >= 2 {
|
|
||||||
return Some(container);
|
|
||||||
}
|
|
||||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
|
||||||
}
|
|
||||||
panic!("postgres container did not become ready");
|
|
||||||
}
|
|
||||||
|
|
||||||
fn psql(container: &DockerPostgres, sql: &str) {
|
|
||||||
let output = Command::new("docker")
|
|
||||||
.args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-c", sql])
|
|
||||||
.output()
|
|
||||||
.expect("run psql");
|
|
||||||
assert!(output.status.success(), "psql failed: {}", String::from_utf8_lossy(&output.stderr));
|
|
||||||
}
|
|
||||||
|
|
||||||
fn postgres_test_config(id: &str, port: u16) -> ConnectionConfig {
|
|
||||||
ConnectionConfig {
|
|
||||||
docs_notes_path: None,
|
|
||||||
id: id.to_string(),
|
|
||||||
name: id.to_string(),
|
|
||||||
note: String::new(),
|
|
||||||
db_type: DatabaseType::Postgres,
|
|
||||||
driver_profile: None,
|
|
||||||
driver_label: None,
|
|
||||||
url_params: None,
|
|
||||||
agent_java_options: Vec::new(),
|
|
||||||
host: "127.0.0.1".to_string(),
|
|
||||||
port,
|
|
||||||
username: "postgres".to_string(),
|
|
||||||
password: "postgres".to_string(),
|
|
||||||
database: Some("postgres".to_string()),
|
|
||||||
default_schema: None,
|
|
||||||
visible_databases: None,
|
|
||||||
visible_schemas: None,
|
|
||||||
attached_databases: Vec::new(),
|
|
||||||
init_script: None,
|
|
||||||
color: None,
|
|
||||||
transport_layers: Vec::new(),
|
|
||||||
connect_timeout_secs: 5,
|
|
||||||
query_timeout_secs: 30,
|
|
||||||
idle_timeout_secs: 60,
|
|
||||||
keepalive_interval_secs: 0,
|
|
||||||
ssl: false,
|
|
||||||
ca_cert_path: String::new(),
|
|
||||||
client_cert_path: String::new(),
|
|
||||||
client_key_path: String::new(),
|
|
||||||
sysdba: false,
|
|
||||||
oracle_connection_type: None,
|
|
||||||
connection_string: None,
|
|
||||||
redis_connection_mode: None,
|
|
||||||
redis_sentinel_master: String::new(),
|
|
||||||
redis_sentinel_nodes: String::new(),
|
|
||||||
redis_sentinel_username: String::new(),
|
|
||||||
redis_sentinel_password: String::new(),
|
|
||||||
redis_sentinel_tls: false,
|
|
||||||
redis_cluster_nodes: String::new(),
|
|
||||||
redis_key_separator: dbx_core::models::connection::default_redis_key_separator(),
|
|
||||||
redis_scan_page_size: None,
|
|
||||||
redis_database_aliases: Default::default(),
|
|
||||||
etcd_endpoints: String::new(),
|
|
||||||
gbase_server: String::new(),
|
|
||||||
informix_server: String::new(),
|
|
||||||
external_config: None,
|
|
||||||
jdbc_driver_class: None,
|
|
||||||
jdbc_driver_paths: Vec::new(),
|
|
||||||
one_time: false,
|
|
||||||
save_password: true,
|
|
||||||
read_only: false,
|
|
||||||
is_production: false,
|
|
||||||
production_databases: vec![],
|
|
||||||
show_system_schemas: false,
|
|
||||||
database_info: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn database_export_writes_structure_and_data_for_all_tables() {
|
fn database_export_writes_structure_and_data_for_all_tables() {
|
||||||
@@ -162,7 +27,7 @@ fn database_export_writes_structure_and_data_for_all_tables() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn run_database_export_writes_structure_and_data_for_all_tables() {
|
async fn run_database_export_writes_structure_and_data_for_all_tables() {
|
||||||
let Some(container) = start_docker_postgres() else {
|
let Some(container) = start_docker_postgres("dbx-export-prefetch") else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,191 @@
|
|||||||
|
//! Shared Docker-backed PostgreSQL test scaffolding for integration tests
|
||||||
|
//! that need a real, disposable server (not a mock) to exercise driver SQL
|
||||||
|
//! against. Needs local Docker; callers should check `docker_ready()` (via
|
||||||
|
//! `start_docker_postgres`, which does this for them) and skip cleanly when
|
||||||
|
//! it's unavailable rather than failing the whole test run.
|
||||||
|
|
||||||
|
use std::process::Command;
|
||||||
|
|
||||||
|
use dbx_core::models::connection::{ConnectionConfig, DatabaseType};
|
||||||
|
|
||||||
|
pub struct DockerPostgres {
|
||||||
|
pub name: String,
|
||||||
|
pub port: u16,
|
||||||
|
/// `false` when this wraps an already-running container the caller
|
||||||
|
/// provided via `DBX_TEST_POSTGRES_CONTAINER` (see
|
||||||
|
/// `start_docker_postgres_with_server_args`) instead of one this helper
|
||||||
|
/// started itself — that container is the caller's to manage, not ours
|
||||||
|
/// to tear down.
|
||||||
|
remove_on_drop: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for DockerPostgres {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if self.remove_on_drop {
|
||||||
|
let _ = Command::new("docker").args(["rm", "-f", &self.name]).status();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn docker_ready() -> bool {
|
||||||
|
Command::new("docker")
|
||||||
|
.args(["version", "--format", "{{.Server.Version}}"])
|
||||||
|
.output()
|
||||||
|
.map(|output| output.status.success())
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Starts a disposable `postgres:16-alpine` container named
|
||||||
|
/// `{name_prefix}-{random uuid}`, waiting for it to accept connections.
|
||||||
|
/// Returns `None` (rather than panicking) when Docker itself isn't
|
||||||
|
/// available, so tests can skip cleanly in environments without it.
|
||||||
|
pub fn start_docker_postgres(name_prefix: &str) -> Option<DockerPostgres> {
|
||||||
|
start_docker_postgres_with_server_args(name_prefix, &[])
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Same as `start_docker_postgres`, but forwards extra `-c name=value`
|
||||||
|
/// postmaster arguments — e.g. `&["-c", "allow_system_table_mods=on"]` to let
|
||||||
|
/// a test create a stand-in `pg_catalog` table for a system catalog this
|
||||||
|
/// image doesn't actually have (like OpenTenBase/Cloudberry-only ones).
|
||||||
|
///
|
||||||
|
/// If `DBX_TEST_POSTGRES_CONTAINER`/`DBX_TEST_POSTGRES_PORT` are set, reuses
|
||||||
|
/// that already-running container instead of starting (and later tearing
|
||||||
|
/// down) a fresh one — handy for iterating on a single test locally without
|
||||||
|
/// paying container startup cost every run. `server_args` is ignored in that
|
||||||
|
/// case, since the caller's container is already configured however they set
|
||||||
|
/// it up.
|
||||||
|
pub fn start_docker_postgres_with_server_args(name_prefix: &str, server_args: &[&str]) -> Option<DockerPostgres> {
|
||||||
|
if let Ok(name) = std::env::var("DBX_TEST_POSTGRES_CONTAINER") {
|
||||||
|
let port = std::env::var("DBX_TEST_POSTGRES_PORT")
|
||||||
|
.expect("DBX_TEST_POSTGRES_PORT must accompany DBX_TEST_POSTGRES_CONTAINER")
|
||||||
|
.parse()
|
||||||
|
.expect("DBX_TEST_POSTGRES_PORT must be a valid port");
|
||||||
|
return Some(DockerPostgres { name, port, remove_on_drop: false });
|
||||||
|
}
|
||||||
|
if !docker_ready() {
|
||||||
|
eprintln!("skipping docker-backed test because Docker is unavailable");
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let port = portpicker::pick_unused_port().expect("pick unused postgres port");
|
||||||
|
let container =
|
||||||
|
DockerPostgres { name: format!("{name_prefix}-{}", uuid::Uuid::new_v4()), port, remove_on_drop: true };
|
||||||
|
|
||||||
|
let mut args = vec![
|
||||||
|
"run".to_string(),
|
||||||
|
"-d".to_string(),
|
||||||
|
"--rm".to_string(),
|
||||||
|
"--name".to_string(),
|
||||||
|
container.name.clone(),
|
||||||
|
"-e".to_string(),
|
||||||
|
"POSTGRES_PASSWORD=postgres".to_string(),
|
||||||
|
"-e".to_string(),
|
||||||
|
"POSTGRES_USER=postgres".to_string(),
|
||||||
|
"-e".to_string(),
|
||||||
|
"POSTGRES_DB=postgres".to_string(),
|
||||||
|
"-p".to_string(),
|
||||||
|
format!("{port}:5432"),
|
||||||
|
"postgres:16-alpine".to_string(),
|
||||||
|
];
|
||||||
|
args.extend(server_args.iter().map(|arg| arg.to_string()));
|
||||||
|
|
||||||
|
let status = Command::new("docker").args(&args).status().expect("start docker postgres");
|
||||||
|
assert!(status.success(), "docker run postgres container should succeed");
|
||||||
|
|
||||||
|
// The postgres image's initdb briefly starts and restarts the server, so
|
||||||
|
// a single successful probe may still land inside that bootstrap window;
|
||||||
|
// require two consecutive successful `SELECT 1`s before calling it ready.
|
||||||
|
let mut consecutive_ok = 0;
|
||||||
|
for _ in 0..120 {
|
||||||
|
let ready = Command::new("docker")
|
||||||
|
.args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-c", "SELECT 1"])
|
||||||
|
.output()
|
||||||
|
.map(|output| output.status.success())
|
||||||
|
.unwrap_or(false);
|
||||||
|
consecutive_ok = if ready { consecutive_ok + 1 } else { 0 };
|
||||||
|
if consecutive_ok >= 2 {
|
||||||
|
return Some(container);
|
||||||
|
}
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(500));
|
||||||
|
}
|
||||||
|
panic!("postgres container did not become ready");
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn psql(container: &DockerPostgres, sql: &str) {
|
||||||
|
let output = Command::new("docker")
|
||||||
|
.args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-c", sql])
|
||||||
|
.output()
|
||||||
|
.expect("run psql");
|
||||||
|
assert!(output.status.success(), "psql failed: {}", String::from_utf8_lossy(&output.stderr));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Each `tests/*.rs` file compiles this module as its own separate crate, so
|
||||||
|
// a helper only some of those test binaries call is "unused" in the others.
|
||||||
|
#[allow(dead_code)]
|
||||||
|
pub fn psql_allow_failure(container: &DockerPostgres, sql: &str) -> bool {
|
||||||
|
Command::new("docker")
|
||||||
|
.args(["exec", &container.name, "psql", "-U", "postgres", "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-c", sql])
|
||||||
|
.output()
|
||||||
|
.map(|output| output.status.success())
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn postgres_test_config(id: &str, port: u16) -> ConnectionConfig {
|
||||||
|
ConnectionConfig {
|
||||||
|
docs_notes_path: None,
|
||||||
|
id: id.to_string(),
|
||||||
|
name: id.to_string(),
|
||||||
|
note: String::new(),
|
||||||
|
db_type: DatabaseType::Postgres,
|
||||||
|
driver_profile: None,
|
||||||
|
driver_label: None,
|
||||||
|
url_params: None,
|
||||||
|
agent_java_options: Vec::new(),
|
||||||
|
host: "127.0.0.1".to_string(),
|
||||||
|
port,
|
||||||
|
username: "postgres".to_string(),
|
||||||
|
password: "postgres".to_string(),
|
||||||
|
database: Some("postgres".to_string()),
|
||||||
|
default_schema: None,
|
||||||
|
visible_databases: None,
|
||||||
|
visible_schemas: None,
|
||||||
|
attached_databases: Vec::new(),
|
||||||
|
init_script: None,
|
||||||
|
color: None,
|
||||||
|
transport_layers: Vec::new(),
|
||||||
|
connect_timeout_secs: 5,
|
||||||
|
query_timeout_secs: 30,
|
||||||
|
idle_timeout_secs: 60,
|
||||||
|
keepalive_interval_secs: 0,
|
||||||
|
ssl: false,
|
||||||
|
ca_cert_path: String::new(),
|
||||||
|
client_cert_path: String::new(),
|
||||||
|
client_key_path: String::new(),
|
||||||
|
sysdba: false,
|
||||||
|
oracle_connection_type: None,
|
||||||
|
connection_string: None,
|
||||||
|
redis_connection_mode: None,
|
||||||
|
redis_sentinel_master: String::new(),
|
||||||
|
redis_sentinel_nodes: String::new(),
|
||||||
|
redis_sentinel_username: String::new(),
|
||||||
|
redis_sentinel_password: String::new(),
|
||||||
|
redis_sentinel_tls: false,
|
||||||
|
redis_cluster_nodes: String::new(),
|
||||||
|
redis_key_separator: dbx_core::models::connection::default_redis_key_separator(),
|
||||||
|
redis_scan_page_size: None,
|
||||||
|
redis_database_aliases: Default::default(),
|
||||||
|
etcd_endpoints: String::new(),
|
||||||
|
gbase_server: String::new(),
|
||||||
|
informix_server: String::new(),
|
||||||
|
external_config: None,
|
||||||
|
jdbc_driver_class: None,
|
||||||
|
jdbc_driver_paths: Vec::new(),
|
||||||
|
one_time: false,
|
||||||
|
save_password: true,
|
||||||
|
read_only: false,
|
||||||
|
is_production: false,
|
||||||
|
production_databases: vec![],
|
||||||
|
show_system_schemas: false,
|
||||||
|
database_info: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user