use regex::Regex; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::sync::Arc; use tokio::sync::RwLock; use crate::connection::{config_for_pool_key, AppState, PoolKind}; use crate::db; use crate::db::agent_driver::AgentTableReadStartParams; use crate::db::mongo_driver::MongoDocumentResult; use crate::models::connection::{ConnectionConfig, DatabaseType}; use crate::object_source_sql::{build_executable_object_source_statements, EditableObjectSourceSqlInput}; use crate::query::{ agent_execute_query_params, pool_error_action, PoolErrorAction, QueryExecutionOptions, AGENT_PROTOCOL_MAX_ROWS, }; use crate::sql::{split_sql_statements, split_sql_statements_for_database}; use crate::sql_dialect::{ normalize_len_params, qualified_transfer_table, quote_transfer_identifier, transfer_column_identifier, }; static CANCELLED: std::sync::LazyLock>> = std::sync::LazyLock::new(|| RwLock::new(HashSet::new())); static OCEANBASE_MYSQL_TABLE_OPTION_RE: std::sync::LazyLock = std::sync::LazyLock::new(|| { Regex::new(r"(?i)\b(?:AUTO_INCREMENT_MODE|REPLICA_NUM|USE_BLOOM_FILTER|TABLET_SIZE|PCTFREE)\s*=") .expect("valid OceanBase MySQL table option regex") }); static MYSQL_COLLATE_CLAUSE_RE: std::sync::LazyLock = std::sync::LazyLock::new(|| { Regex::new(r"(?i)\bCOLLATE\s*=?\s*([A-Za-z0-9_]+)\b").expect("valid MySQL COLLATE clause regex") }); // An inline FK constraint *definition line*: an optional `CONSTRAINT ` // prefix followed by `FOREIGN KEY (`. Anchored to the line start so column // definitions whose COMMENT/DEFAULT strings mention "foreign key" never match. static INLINE_FOREIGN_KEY_CONSTRAINT_LINE_RE: std::sync::LazyLock = std::sync::LazyLock::new(|| { Regex::new(r#"(?i)^\s*(?:CONSTRAINT\s+(?:`[^`]*`|"[^"]*"|\S+)\s+)?FOREIGN\s+KEY\s*\("#) .expect("valid inline foreign key constraint line regex") }); const MAX_TRANSFER_WRITE_SQL_BYTES: usize = 512 * 1024; const MAX_SQLSERVER_INSERT_ROWS: usize = 1000; const MAX_ORACLE_INSERT_ALL_ROWS: usize = 500; const MAX_ORACLE_MERGE_ROWS: usize = 500; const TRANSFER_TARGET_TABLE_LOOKUP_LIMIT: usize = 1000; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct SqlBatchLimits { max_rows: usize, target_sql_bytes: usize, hard_sql_bytes: Option, } impl SqlBatchLimits { pub(crate) fn for_database(db_type: &DatabaseType, requested_max_rows: usize) -> Self { let max_rows = requested_max_rows.max(1).min(match db_type { DatabaseType::SqlServer => MAX_SQLSERVER_INSERT_ROWS, DatabaseType::Oracle => MAX_ORACLE_INSERT_ALL_ROWS, _ => usize::MAX, }); let target_sql_bytes = match db_type { DatabaseType::CloudflareD1 => crate::db::cloudflare_d1::MAX_SQL_STATEMENT_BYTES, _ => MAX_TRANSFER_WRITE_SQL_BYTES, }; Self { max_rows, target_sql_bytes, hard_sql_bytes: None } } pub(crate) fn with_hard_sql_bytes(mut self, hard_sql_bytes: Option) -> Self { self.hard_sql_bytes = hard_sql_bytes; self } } #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] #[serde(rename_all = "camelCase")] pub enum TransferMode { #[default] Append, Overwrite, Upsert, } #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] #[serde(rename_all = "camelCase")] pub enum TransferTableNameCase { #[default] Preserve, Lower, Upper, } #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] #[serde(rename_all = "camelCase")] pub enum TransferOwnershipPolicy { #[default] Preserve, Skip, ReassignMissing, } #[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub enum TransferContent { #[default] StructureAndData, StructureOnly, DataOnly, } #[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq, Hash)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum TransferObjectKind { #[default] Table, View, MaterializedView, Procedure, Function, Trigger, Sequence, Event, } #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] #[serde(rename_all = "camelCase")] pub struct TransferObjectSelection { pub object_type: TransferObjectKind, pub names: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum TransferObjectFamily { Mysql, Postgres, Oracle, SqlServer, } pub fn transfer_object_family(db_type: &DatabaseType) -> Option { match db_type { DatabaseType::Mysql | DatabaseType::Gbase => Some(TransferObjectFamily::Mysql), DatabaseType::Postgres | DatabaseType::Kingbase | DatabaseType::Gaussdb | DatabaseType::Kwdb | DatabaseType::OpenGauss => Some(TransferObjectFamily::Postgres), DatabaseType::Oracle | DatabaseType::Dameng | DatabaseType::OceanbaseOracle => { Some(TransferObjectFamily::Oracle) } DatabaseType::SqlServer => Some(TransferObjectFamily::SqlServer), _ => None, } } pub fn is_same_transfer_family(a: &DatabaseType, b: &DatabaseType) -> bool { match (transfer_object_family(a), transfer_object_family(b)) { (Some(fa), Some(fb)) => fa == fb, _ => false, } } pub fn transfer_object_kinds_for_family(family: &TransferObjectFamily) -> Vec { use TransferObjectKind::*; match family { TransferObjectFamily::Mysql => vec![Table, View, Procedure, Function, Trigger, Event], TransferObjectFamily::Postgres => vec![Table, View, MaterializedView, Procedure, Function, Trigger, Sequence], TransferObjectFamily::Oracle => vec![Table, View, MaterializedView, Procedure, Function, Trigger, Sequence], TransferObjectFamily::SqlServer => vec![Table, View, Procedure, Function, Trigger, Sequence], } } pub fn transfer_object_kinds(db_type: &DatabaseType) -> Vec { match transfer_object_family(db_type) { Some(family) => transfer_object_kinds_for_family(&family), None => Vec::new(), } } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct TransferRequest { pub transfer_id: String, pub source_connection_id: String, pub source_database: String, pub source_schema: String, pub source_catalog: Option, pub target_connection_id: String, pub target_database: String, pub target_schema: String, pub target_catalog: Option, pub tables: Vec, pub create_table: bool, #[serde(default)] pub content: TransferContent, #[serde(default)] pub objects: Vec, #[serde(default)] pub mode: TransferMode, #[serde(default)] pub target_table_name_case: TransferTableNameCase, #[serde(default = "default_quote_target_column_names")] pub quote_target_column_names: bool, #[serde(default)] pub ownership_policy: TransferOwnershipPolicy, pub batch_size: usize, } fn default_quote_target_column_names() -> bool { true } #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct TransferOwnershipPreview { pub missing_owners: Vec, pub target_owner: String, } impl TransferRequest { pub fn target_table_name(&self, source_table: &str) -> String { match self.target_table_name_case { TransferTableNameCase::Preserve => source_table.to_string(), TransferTableNameCase::Lower => source_table.to_lowercase(), TransferTableNameCase::Upper => source_table.to_uppercase(), } } } #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct TransferProgress { pub transfer_id: String, pub table: String, pub table_index: usize, pub total_tables: usize, pub rows_transferred: u64, pub total_rows: Option, pub status: TransferStatus, pub error: Option, pub terminal: bool, } #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub enum TransferStatus { Running, TableDone, Done, Error, Cancelled, } pub fn quote_identifier(name: &str, db_type: &DatabaseType) -> String { quote_transfer_identifier(name, db_type) } fn quote_identifier_with_identifier_quote( name: &str, db_type: &DatabaseType, identifier_quote: Option<&str>, ) -> String { crate::sql_dialect::quote_table_data_identifier(Some(*db_type), name, identifier_quote) } /// Resolve an optional catalog for external-catalog routing in the transfer /// pipeline. Returns `Some(catalog)` only when: /// - the catalog is non-empty and not a built-in catalog name, AND /// - the database type is Doris/StarRocks, or MySQL (StarRocks/Doris are often /// saved as `db_type=mysql` with a matching `driver_profile`; the transfer UI /// only sends `sourceCatalog`/`targetCatalog` for catalog-capable connections). /// /// Built-in catalogs: Doris `internal`, StarRocks `default_catalog`. Prefer /// [`resolve_external_transfer_catalog_for_config`] when a full connection /// config is available (also matches `driver_profile=starrocks|doris`). pub fn resolve_external_transfer_catalog<'a>(catalog: Option<&'a str>, db_type: &DatabaseType) -> Option<&'a str> { let catalog = normalize_external_catalog_name(catalog)?; match db_type { DatabaseType::Doris | DatabaseType::StarRocks | DatabaseType::Mysql => Some(catalog), _ => None, } } /// Like [`resolve_external_transfer_catalog`], but also treats MySQL connections /// with a Doris/StarRocks `driver_profile` as catalog-capable. pub fn resolve_external_transfer_catalog_for_config<'a>( catalog: Option<&'a str>, config: &crate::models::connection::ConnectionConfig, ) -> Option<&'a str> { let catalog = normalize_external_catalog_name(catalog)?; if db::mysql_compatible::supports_external_catalogs(config) { Some(catalog) } else { None } } fn normalize_external_catalog_name(catalog: Option<&str>) -> Option<&str> { let catalog = catalog?.trim(); if catalog.is_empty() || catalog.eq_ignore_ascii_case("internal") || catalog.eq_ignore_ascii_case("default_catalog") { return None; } Some(catalog) } /// Create (or reuse) a transfer pool for the given connection/database/catalog. /// /// For Doris/StarRocks external catalogs the pool is created with /// `catalog=` in the connection URL params so mysql_async runs /// `SET catalog` **before** `USE ` during setup. The handshake does /// not send the external database name (mysql_async strips the path when a /// catalog is present), which is what previously caused `Unknown database`. pub async fn ensure_transfer_pool( state: &AppState, connection_id: &str, database: &str, catalog: Option<&str>, ) -> Result { let config = { let configs = state.configs.read().await; configs.get(connection_id).cloned().ok_or_else(|| format!("Connection config not found: {connection_id}"))? }; if let Some(catalog) = resolve_external_transfer_catalog_for_config(catalog, &config) { // SET catalog first, then USE so session has both catalog and // database selected — StarRocks rejects unqualified analysis with // "No database selected" when only SET catalog ran. state.get_or_create_pool_with_catalog(connection_id, Some(database), Some(catalog)).await } else { state.get_or_create_pool(connection_id, Some(database)).await } } pub fn qualified_table(table: &str, schema: &str, db_type: &DatabaseType, catalog: Option<&str>) -> String { // Only use 3-part catalog-qualified names for Doris/StarRocks external catalogs. let effective_catalog = resolve_external_transfer_catalog(catalog, db_type); qualified_transfer_table(table, schema, db_type, effective_catalog) } fn qualified_table_with_identifier_quote( table: &str, schema: &str, db_type: &DatabaseType, catalog: Option<&str>, identifier_quote: Option<&str>, ) -> String { if crate::sql_dialect::uses_connection_identifier_quote(Some(*db_type), identifier_quote) { crate::sql_dialect::table_data_qualified_table_name( Some(*db_type), (!schema.trim().is_empty()).then_some(schema), table, identifier_quote, ) } else { qualified_table(table, schema, db_type, catalog) } } pub fn validate_transfer_target_table_names(request: &TransferRequest) -> Result<(), String> { let mut targets: HashMap = HashMap::new(); for source_table in &request.tables { let target_table = request.target_table_name(source_table); if let Some(first_source) = targets.insert(target_table.clone(), source_table.clone()) { return Err(format!( "Target table name collision after case conversion: '{first_source}' and '{source_table}' both map to '{target_table}'" )); } } Ok(()) } /// Kinds that may be transferred between different database families. /// Only DDL shapes that can be mechanically rewritten are allowed: /// views (CREATE ... VIEW ... AS SELECT) and sequences (CREATE SEQUENCE). /// Sequences additionally require both sides to support the type (MySQL does not). /// Cross-family transfer is only supported between the MySQL, SQL Server and /// Oracle/Dameng families; the Postgres family is not a validated source or /// target for the cross-family DDL pipeline (the executor rejects it). pub fn cross_family_transferable_object_kinds(source: &DatabaseType, target: &DatabaseType) -> Vec { use TransferObjectKind::*; if is_same_transfer_family(source, target) { return transfer_object_kinds(source); } // Narrow the matrix to validated directions: MySQL, SQL Server and // Oracle/Dameng may act as either side. Postgres (and anything else) is // excluded — the executor rejects Postgres sources and no dialect-aware // conversion is validated for it. let family_supported = |db_type: &DatabaseType| { matches!( transfer_object_family(db_type), Some(TransferObjectFamily::Mysql) | Some(TransferObjectFamily::SqlServer) | Some(TransferObjectFamily::Oracle) ) }; if !family_supported(source) || !family_supported(target) { return Vec::new(); } // Cross-family VIEW transfer is disabled: convert_cross_family_object_ddl // only rewrites the DDL wrapper, identifier quoting and schema qualifiers — // it does not translate the view query body, so source-specific constructs // (IFNULL, TOP, GETDATE, …) would execute unchanged on an incompatible // target. Only sequences are allowed: their CREATE SEQUENCE statements are // plain DDL (no query body) and the small set of dialect differences // (AS , NOCYCLE/NOCACHE) is converted and tested. let source_kinds = transfer_object_kinds(source); let target_kinds = transfer_object_kinds(target); let mut allowed = Vec::new(); if source_kinds.contains(&Sequence) && target_kinds.contains(&Sequence) { allowed.push(Sequence); } allowed } /// Rewrites a source DDL fragment to the target family's quoting style and /// schema qualifier. Prefix normalization (DEFINER/ALGORITHM/FORCE/WITH /// SCHEMABINDING/AS ) is applied per source family. pub fn convert_cross_family_object_ddl( source_family: &TransferObjectFamily, target_family: &TransferObjectFamily, kind: &TransferObjectKind, source_schema: &str, target_schema: &str, ddl: &str, ) -> String { let mut sql = ddl.trim().to_string(); match kind { TransferObjectKind::View => match source_family { TransferObjectFamily::Mysql => { sql = strip_mysql_definer(&sql); sql = strip_sql_view_prefix(&sql); } TransferObjectFamily::Oracle => { sql = strip_sql_view_prefix(&sql); } TransferObjectFamily::SqlServer => { sql = strip_sqlserver_view_with_clause(&sql); } _ => {} }, TransferObjectKind::Sequence => match source_family { TransferObjectFamily::SqlServer => { sql = strip_sqlserver_sequence_as_type(&sql); } TransferObjectFamily::Oracle if target_family == &TransferObjectFamily::SqlServer => { sql = sql.replace("NOCYCLE", "NO CYCLE").replace("NOCACHE", "NO CACHE"); } _ => {} }, _ => {} } // Mask string literals and comments before any identifier/schema // rewrite, then restore them afterwards: regexes that re-quote // identifiers must never touch text inside strings or comments. MySQL // double-quoted text is a string literal (unless ANSI_QUOTES is on). let (masked, restores) = protect_sql_literals(&sql, matches!(source_family, TransferObjectFamily::Mysql)); sql = rewrite_identifiers_to_target(&masked, target_family); if !source_schema.is_empty() && !target_schema.is_empty() && !source_schema.eq_ignore_ascii_case(target_schema) { sql = rewrite_cross_family_schema_qualifier(&sql, target_family, source_schema, target_schema); } if kind == &TransferObjectKind::View { sql = qualify_cross_family_view_target(&sql, target_family, target_schema); } for (placeholder, original) in restores { sql = sql.replace(&placeholder, &original); } sql } /// Locates string literals and comments in SQL: single-quoted strings /// (with `''` and backslash escapes), MySQL double-quoted strings when /// `double_quote_is_string` is set, `--`/`#` line comments and `/* */` /// block comments. Returns byte ranges `(start, end)` of those spans. fn sql_non_code_spans_with_mysql_identifiers( sql: &str, double_quote_is_string: bool, backtick_is_identifier: bool, ) -> Vec<(usize, usize)> { let bytes = sql.as_bytes(); let mut spans = Vec::new(); let mut i = 0; while i < bytes.len() { let b = bytes[i]; let starts_non_code = match b { b'\'' => true, b'"' if double_quote_is_string => true, b'`' if backtick_is_identifier => true, b'-' if i + 1 < bytes.len() && bytes[i + 1] == b'-' => true, b'#' => true, // MySQL line comment b'/' if i + 1 < bytes.len() && bytes[i + 1] == b'*' => true, _ => false, }; if !starts_non_code { i += 1; continue; } let start = i; i = match b { b'\'' | b'"' | b'`' => { i += 1; while i < bytes.len() { if bytes[i] == b'\\' && i + 1 < bytes.len() { i += 2; // backslash escape (MySQL style) continue; } if bytes[i] == b && i + 1 < bytes.len() && bytes[i + 1] == b { i += 2; // '' / "" doubled quote continue; } if bytes[i] == b { i += 1; break; } i += 1; } i } b'-' | b'#' => { while i < bytes.len() && bytes[i] != b'\n' { i += 1; } i } _ => { i += 2; // /* ... */ while i + 1 < bytes.len() && !(bytes[i] == b'*' && bytes[i + 1] == b'/') { i += 1; } (i + 2).min(bytes.len()) } }; spans.push((start, i)); } spans } fn sql_non_code_spans(sql: &str, double_quote_is_string: bool) -> Vec<(usize, usize)> { sql_non_code_spans_with_mysql_identifiers(sql, double_quote_is_string, false) } /// Applies `f` to every code span of `sql`; string literals and comments /// (see `sql_non_code_spans`) pass through verbatim so rewrites never touch /// text inside them. fn map_sql_code_spans(sql: &str, double_quote_is_string: bool, mut f: F) -> String where F: FnMut(&str) -> String, { let spans = sql_non_code_spans(sql, double_quote_is_string); let mut out = String::with_capacity(sql.len()); let mut prev = 0; for (start, end) in spans { if start > prev { out.push_str(&f(&sql[prev..start])); } out.push_str(&sql[start..end]); prev = end; } if prev < sql.len() { out.push_str(&f(&sql[prev..])); } out } fn map_mysql_ddl_code_spans(sql: &str, mut f: F) -> String where F: FnMut(&str) -> String, { let spans = sql_non_code_spans_with_mysql_identifiers(sql, true, true); let mut out = String::with_capacity(sql.len()); let mut prev = 0; for (start, end) in spans { if start > prev { out.push_str(&f(&sql[prev..start])); } out.push_str(&sql[start..end]); prev = end; } if prev < sql.len() { out.push_str(&f(&sql[prev..])); } out } /// Masks string literals and comments with placeholders so subsequent regex /// rewrites cannot touch them, and returns the restore map to swap the /// original text back afterwards. fn protect_sql_literals(sql: &str, double_quote_is_string: bool) -> (String, Vec<(String, String)>) { let spans = sql_non_code_spans(sql, double_quote_is_string); let mut out = String::with_capacity(sql.len()); let mut restores = Vec::new(); let mut prev = 0; for (start, end) in spans { if start > prev { out.push_str(&sql[prev..start]); } let placeholder = format!("__DBX_LIT_{}__", restores.len()); out.push_str(&placeholder); restores.push((placeholder, sql[start..end].to_string())); prev = end; } if prev < sql.len() { out.push_str(&sql[prev..]); } (out, restores) } /// Aligns a cross-family view DDL with the target schema: the `CREATE VIEW` /// target and bare table references in the body (`FROM`/`JOIN`/`INTO`/`UPDATE`) /// are qualified with the target schema, matching how table transfer creates /// tables (`"schema"."table"`). References that already carry a prefix are left /// untouched. String literals and comments are preserved verbatim (see /// `map_sql_code_spans`). fn qualify_cross_family_view_target(sql: &str, target_family: &TransferObjectFamily, target_schema: &str) -> String { if target_schema.is_empty() { return sql.to_string(); } let (open, close) = match target_family { TransferObjectFamily::Mysql => ("`", "`"), TransferObjectFamily::SqlServer => ("[", "]"), TransferObjectFamily::Oracle => ("\"", "\""), _ => return sql.to_string(), }; let qo = regex::escape(open); let qc = regex::escape(close); // Identifiers may contain any characters (including CJK) except the // closing quote of the target dialect, since every identifier has been // normalized to the target quoting style by this point. let ident = match target_family { TransferObjectFamily::Mysql => r#"[^`]+"#, TransferObjectFamily::SqlServer => r#"[^\]]+"#, TransferObjectFamily::Oracle => r#"[^"]+"#, _ => "[A-Za-z_][A-Za-z0-9_]*", }; let create_re = Regex::new(&format!(r"(?i)\bCREATE\s+VIEW\s+(?:{qo}({ident}){qc}\.)?{qo}({ident}){qc}")).unwrap(); let mut out = create_re .replace_all(sql, |caps: ®ex::Captures| { let name = &caps[2]; if matches!(caps.get(1), Some(m) if !m.as_str().is_empty()) { caps[0].to_string() } else { format!("CREATE VIEW {open}{target_schema}{close}.{open}{name}{close}") } }) .to_string(); // bare references after FROM/JOIN/INTO/UPDATE/TABLE get the schema prefix; // already-prefixed references (group 1) stay as-is let ref_re = Regex::new(&format!(r"(?i)\b(from|join|into|update|table)\s+(?:{qo}({ident}){qc}\.)?{qo}({ident}){qc}")) .unwrap(); out = ref_re .replace_all(&out, |caps: ®ex::Captures| { let name = &caps[3]; if matches!(caps.get(2), Some(m) if !m.as_str().is_empty()) { caps[0].to_string() } else { format!("{} {open}{target_schema}{close}.{open}{name}{close}", &caps[1]) } }) .to_string(); out } /// Collapses `CREATE [ALGORITHM=..] [DEFINER=..] [SQL SECURITY ..] VIEW`, /// `CREATE OR REPLACE [FORCE] [NONEDITIONABLE] VIEW` and plain `CREATE VIEW` /// into a bare `CREATE VIEW` prefix usable on every target family. fn strip_sql_view_prefix(sql: &str) -> String { if let Some(pos) = sql.find(" VIEW ") { let head = &sql[..pos]; if head.trim_start().starts_with("CREATE") { return format!("CREATE VIEW{}", &sql[pos + 5..]); } } sql.to_string() } /// Removes `WITH SCHEMABINDING` between the view name and `AS` in SQL Server /// view definitions. fn strip_sqlserver_view_with_clause(sql: &str) -> String { let re = Regex::new(r"(?i)\s+WITH\s+SCHEMABINDING\s+").unwrap(); map_sql_code_spans(sql, false, |code| re.replace_all(code, " ").to_string()) } /// Removes the ` AS ` clause from a SQL Server CREATE SEQUENCE so the /// DDL fits Dameng/Oracle sequence syntax. fn strip_sqlserver_sequence_as_type(sql: &str) -> String { let re = Regex::new(r"(?i)\s+AS\s+(?:BIGINT|INT|SMALLINT|TINYINT|DECIMAL\s*\([^)]*\)|NUMERIC\s*\([^)]*\))\s+").unwrap(); map_sql_code_spans(sql, false, |code| re.replace_all(code, " ").to_string()) } /// Rewrites backtick / double-quote / bracket identifier quoting to the /// target family's style. Only identifiers in code positions are rewritten: /// string literals and comments are preserved verbatim (see /// `map_sql_code_spans`). The source family decides whether double-quoted /// text is a string literal (MySQL, unless ANSI_QUOTES is on) or an /// identifier (SQL Server / Oracle). fn rewrite_identifiers_to_target(sql: &str, target: &TransferObjectFamily) -> String { let (open, close, pattern) = match target { TransferObjectFamily::Mysql => ("`", "`", r#""([^"]+)"|\[([^\]]+)\]"#), TransferObjectFamily::SqlServer => ("[", "]", r#"`([^`]+)`|"([^"]+)""#), TransferObjectFamily::Oracle => ("\"", "\"", r#"`([^`]+)`|\[([^\]]+)\]"#), _ => return sql.to_string(), }; let re = Regex::new(pattern).unwrap(); re.replace_all(sql, |caps: ®ex::Captures| { let name = caps.get(1).map(|m| m.as_str()).or_else(|| caps.get(2).map(|m| m.as_str())).unwrap_or(""); format!("{open}{name}{close}") }) .to_string() } /// Rewrites `{quote}{source_schema}{quote}.` qualifiers to the target schema /// in the target family's quoting style. fn rewrite_cross_family_schema_qualifier( sql: &str, target: &TransferObjectFamily, source_schema: &str, target_schema: &str, ) -> String { let (open, close) = match target { TransferObjectFamily::Mysql => ("`", "`"), TransferObjectFamily::SqlServer => ("[", "]"), TransferObjectFamily::Oracle => ("\"", "\""), _ => return sql.to_string(), }; let source = format!("{open}{}{close}.", regex::escape(source_schema)); let target = format!("{open}{target_schema}{close}."); sql.replace(&source, &target) } pub fn validate_transfer_request(request: &TransferRequest) -> Result<(), String> { validate_transfer_target_table_names(request)?; if matches!(request.content, TransferContent::DataOnly) && !request.objects.is_empty() { return Err("仅数据模式不传输非表对象".to_string()); } for selection in &request.objects { if selection.names.is_empty() { return Err(format!("Object selection for {:?} is empty", selection.object_type)); } for name in &selection.names { if name.trim().is_empty() || name.contains('\0') { return Err(format!("Invalid object name: {name:?}")); } } } Ok(()) } #[derive(Debug, Clone, PartialEq, Eq)] struct ResolvedTransferTargetTable { name: String, preexisting: bool, } fn json_scalar_to_string(value: &serde_json::Value) -> Option { match value { serde_json::Value::String(value) => Some(value.clone()), serde_json::Value::Number(value) => Some(value.to_string()), serde_json::Value::Bool(value) => Some(value.to_string()), _ => None, } } fn mysql_lower_case_table_names_from_result(result: &db::QueryResult) -> Option { let row = result.rows.first()?; row.get(1).or_else(|| row.first()).and_then(json_scalar_to_string)?.trim().parse::().ok() } async fn target_table_lookup_is_case_insensitive( state: &AppState, target_pool_key: &str, target_db_type: &DatabaseType, ) -> bool { if !matches!(target_db_type, DatabaseType::Mysql) { return false; } let result = match execute_on_pool(state, target_pool_key, "SHOW VARIABLES LIKE 'lower_case_table_names'").await { Ok(result) => result, Err(error) => { log::debug!("[transfer] failed to read MySQL lower_case_table_names: {error}"); return false; } }; // MySQL lower_case_table_names=1/2 means table lookup is case-insensitive. // Prefer the metadata name so generated INSERT/TRUNCATE SQL keeps the target // table's declared case instead of the source-derived request case. mysql_lower_case_table_names_from_result(&result).is_some_and(|value| value != 0) } fn existing_transfer_target_table_name( requested_name: &str, tables: &[db::TableInfo], allow_case_insensitive_match: bool, ) -> Option { if let Some(table) = tables.iter().find(|table| table.name == requested_name) { return Some(table.name.clone()); } if !allow_case_insensitive_match { return None; } tables.iter().find(|table| table.name.eq_ignore_ascii_case(requested_name)).map(|table| table.name.clone()) } const TRANSFER_PROGRESS_CHANNEL_CAPACITY: usize = 16; fn try_send_transfer_progress(sender: &tokio::sync::mpsc::Sender, progress: TransferProgress) { let _ = sender.try_send(progress); } struct AbortTransferTaskOnDrop(tokio::task::AbortHandle); impl Drop for AbortTransferTaskOnDrop { fn drop(&mut self) { self.0.abort(); } } #[allow(clippy::too_many_arguments)] async fn list_transfer_tables_isolated( state: Arc, connection_id: String, database: String, schema: String, catalog: Option, db_type: DatabaseType, table: String, limit: usize, ) -> Result, String> { let task = tokio::spawn(async move { if let Some(catalog) = resolve_external_transfer_catalog(catalog.as_deref(), &db_type) { crate::schema::list_doris_catalog_tables_core( &state, &connection_id, catalog, &database, Some(&table), Some(limit), None, None, None, ) .await } else { crate::schema::list_tables_core( &state, &connection_id, &database, &schema, Some(&table), Some(limit), None, None, None, ) .await } }); let _abort_on_drop = AbortTransferTaskOnDrop(task.abort_handle()); task.await.map_err(|error| format!("Transfer table metadata task failed: {error}"))? } async fn resolve_transfer_target_table_name( state: &Arc, request: &TransferRequest, source_table: &str, target_pool_key: &str, target_db_type: &DatabaseType, _source_catalog: Option<&str>, target_catalog: Option<&str>, ) -> ResolvedTransferTargetTable { let requested_name = request.target_table_name(source_table); if is_mongodb_transfer_type(target_db_type) { return ResolvedTransferTargetTable { name: requested_name, preexisting: false }; } let allow_case_insensitive_match = target_table_lookup_is_case_insensitive(state, target_pool_key, target_db_type).await; // Route through the catalog-aware path when targeting an external // Doris/StarRocks catalog — otherwise the lookup runs against the // default / internal catalog and can miss or misidentify the table. let tables = list_transfer_tables_isolated( state.clone(), request.target_connection_id.clone(), request.target_database.clone(), request.target_schema.clone(), target_catalog.map(str::to_string), *target_db_type, requested_name.clone(), TRANSFER_TARGET_TABLE_LOOKUP_LIMIT, ) .await .unwrap_or_else(|error| { log::debug!("[transfer] failed to resolve target table metadata for {requested_name}: {error}"); Vec::new() }); if let Some(existing_name) = existing_transfer_target_table_name(&requested_name, &tables, allow_case_insensitive_match) { ResolvedTransferTargetTable { name: existing_name, preexisting: true } } else { ResolvedTransferTargetTable { name: requested_name, preexisting: false } } } fn quote_string_literal(value: &str) -> String { format!("'{}'", value.replace('\'', "''")) } /// Returns a SQL statement selecting 1 row when `name` exists in `schema` /// on the target side; None-kind support per family mirrors /// `transfer_object_kinds`. pub fn target_object_exists_sql( db_type: &DatabaseType, schema: &str, name: &str, kind: &TransferObjectKind, ) -> Result { let schema = quote_string_literal(schema); let name = quote_string_literal(name); let q = |literal: &str| literal.to_string(); let sql = match (transfer_object_family(db_type), kind) { (Some(TransferObjectFamily::Mysql), TransferObjectKind::Table | TransferObjectKind::View) => format!( "SELECT 1 FROM information_schema.TABLES WHERE TABLE_SCHEMA = {schema} AND TABLE_NAME = {name} \ AND TABLE_TYPE {} 'VIEW'", if matches!(kind, TransferObjectKind::View) { "=" } else { "<>" } ), (Some(TransferObjectFamily::Mysql), TransferObjectKind::Procedure | TransferObjectKind::Function) => format!( "SELECT 1 FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA = {schema} AND ROUTINE_NAME = {name} \ AND ROUTINE_TYPE = {}", q(if matches!(kind, TransferObjectKind::Procedure) { "'PROCEDURE'" } else { "'FUNCTION'" }) ), (Some(TransferObjectFamily::Mysql), TransferObjectKind::Trigger) => format!( "SELECT 1 FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA = {schema} AND TRIGGER_NAME = {name}" ), (Some(TransferObjectFamily::Mysql), TransferObjectKind::Event) => { format!("SELECT 1 FROM information_schema.EVENTS WHERE EVENT_SCHEMA = {schema} AND EVENT_NAME = {name}") } (Some(TransferObjectFamily::Postgres), TransferObjectKind::View | TransferObjectKind::MaterializedView) => { format!( "SELECT 1 FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ WHERE n.nspname = {schema} AND c.relname = {name} AND c.relkind {}", if matches!(kind, TransferObjectKind::MaterializedView) { "= 'm'" } else { "= 'v'" } ) } (Some(TransferObjectFamily::Postgres), TransferObjectKind::Table) => format!( "SELECT 1 FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ WHERE n.nspname = {schema} AND c.relname = {name} AND c.relkind = 'r'" ), (Some(TransferObjectFamily::Postgres), TransferObjectKind::Procedure | TransferObjectKind::Function) => { format!( "SELECT 1 FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace \ WHERE n.nspname = {schema} AND p.proname = {name}" ) } (Some(TransferObjectFamily::Postgres), TransferObjectKind::Sequence) => format!( "SELECT 1 FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ WHERE n.nspname = {schema} AND c.relname = {name} AND c.relkind = 'S'" ), (Some(TransferObjectFamily::Postgres), TransferObjectKind::Trigger) => format!( "SELECT 1 FROM pg_catalog.pg_trigger t JOIN pg_catalog.pg_class c ON c.oid = t.tgrelid \ JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ WHERE n.nspname = {schema} AND t.tgname = {name} AND NOT t.tgisinternal" ), ( Some(TransferObjectFamily::Oracle), TransferObjectKind::View | TransferObjectKind::MaterializedView | TransferObjectKind::Procedure | TransferObjectKind::Function | TransferObjectKind::Trigger | TransferObjectKind::Sequence, ) => format!( "SELECT 1 FROM ALL_OBJECTS WHERE OWNER = {schema} AND OBJECT_NAME = {name} AND OBJECT_TYPE = {}", q(match kind { TransferObjectKind::View => "'VIEW'", TransferObjectKind::MaterializedView => "'MATERIALIZED VIEW'", TransferObjectKind::Procedure => "'PROCEDURE'", TransferObjectKind::Function => "'FUNCTION'", TransferObjectKind::Trigger => "'TRIGGER'", _ => "'SEQUENCE'", }) ), ( Some(TransferObjectFamily::SqlServer), TransferObjectKind::Table | TransferObjectKind::View | TransferObjectKind::Procedure | TransferObjectKind::Function | TransferObjectKind::Trigger | TransferObjectKind::Sequence, ) => format!( "SELECT 1 FROM sys.objects o JOIN sys.schemas s ON s.schema_id = o.schema_id \ WHERE s.name = {schema} AND o.name = {name} AND o.type IN ({})", q(match kind { TransferObjectKind::Table => "'U'", TransferObjectKind::View => "'V'", TransferObjectKind::Procedure => "'P'", TransferObjectKind::Function => "'FN','IF','TF','FS','FT'", TransferObjectKind::Trigger => "'TR'", _ => "'SO'", }) ), _ => return Err(format!("Object existence check not supported for {:?} {:?}", db_type, kind)), }; Ok(sql) } /// Remove `DEFINER=`user`@`host`` tokens from MySQL DDL (they are not /// transferable and frequently reference accounts that don't exist on target). pub fn strip_mysql_definer(ddl: &str) -> String { let re = Regex::new(r"(?i)\bDEFINER\s*=\s*`[^`]*`@`[^`]*`\s*").unwrap(); re.replace_all(ddl, "").to_string() } /// Rewrite backtick-qualified `schema`.`name` references from `source_schema` /// to `target_schema` in MySQL DDL. pub fn rewrite_mysql_schema_qualifier(ddl: &str, source_schema: &str, target_schema: &str) -> String { if source_schema == target_schema || source_schema.is_empty() { return ddl.to_string(); } let re = Regex::new(&format!(r"`{}`\.", regex::escape(source_schema))).unwrap(); re.replace_all(ddl, &format!("`{}`.", target_schema)).to_string() } pub fn mysql_trigger_ddl( schema: &str, name: &str, timing: &str, manipulation: &str, table: &str, statement: &str, ) -> String { format!( "CREATE TRIGGER `{name}` {timing} {manipulation} ON `{schema}`.`{table}` FOR EACH ROW {statement}", name = name, timing = timing, manipulation = manipulation, schema = schema, table = table, statement = statement.trim() ) } pub fn mysql_event_ddl(_schema: &str, name: &str, status: &str, schedule: &str, body: &str) -> String { format!( "CREATE EVENT `{name}` ON SCHEDULE {schedule} {status} DO {body}", name = name, schedule = schedule, status = status, body = body.trim() ) } /// Builds the query that fetches DDL for one MySQL object. /// - View/Procedure/Function → `SHOW CREATE ...` /// - Trigger → information_schema.TRIGGERS row (timing/manipulation/table/ /// statement) via `mysql_trigger_ddl`. /// - Event → information_schema.EVENTS row via `mysql_event_ddl`. pub fn mysql_object_source_query(kind: &TransferObjectKind, database: &str, name: &str) -> Result { let db = quote_string_literal(database); let n = quote_string_literal(name); let ddl = match kind { TransferObjectKind::View => format!("SHOW CREATE VIEW `{database}`.`{name}`"), TransferObjectKind::Procedure => format!("SHOW CREATE PROCEDURE `{database}`.`{name}`"), TransferObjectKind::Function => format!("SHOW CREATE FUNCTION `{database}`.`{name}`"), TransferObjectKind::Trigger => format!( "SELECT TRIGGER_NAME, ACTION_TIMING, EVENT_MANIPULATION, EVENT_OBJECT_TABLE, ACTION_STATEMENT \ FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA = {db} AND TRIGGER_NAME = {n}" ), TransferObjectKind::Event => format!( "SELECT EVENT_NAME, STATUS, EXECUTE_AT, INTERVAL_VALUE, INTERVAL_FIELD, EVENT_DEFINITION \ FROM information_schema.EVENTS WHERE EVENT_SCHEMA = {db} AND EVENT_NAME = {n}" ), _ => return Err(format!("MySQL object source not supported for {:?}", kind)), }; Ok(ddl) } /// Maps a MySQL DDL query result row to a single DDL string. /// SHOW CREATE column index: view=1, routine=2 (same convention as /// `schema::mysql_object_source_ddl_column_index`); triggers and events are /// assembled from information_schema cells. pub fn mysql_object_ddl_from_result( kind: &TransferObjectKind, database: &str, rows: &[Vec], ) -> Result { let row = rows.first().ok_or_else(|| format!("No rows returned for MySQL {:?} DDL", kind))?; let cell = |idx: usize| -> Result<&str, String> { row.get(idx) .and_then(|value| value.as_str()) .ok_or_else(|| format!("Missing column {idx} in MySQL {:?} DDL result", kind)) }; match kind { TransferObjectKind::View => Ok(cell(1)?.to_string()), TransferObjectKind::Procedure | TransferObjectKind::Function => Ok(cell(2)?.to_string()), TransferObjectKind::Trigger => { let name = cell(0)?; let timing = cell(1)?; let manipulation = cell(2)?; let table = cell(3)?; let statement = cell(4)?; Ok(mysql_trigger_ddl(database, name, timing, manipulation, table, statement)) } TransferObjectKind::Event => { let name = cell(0)?; let status = cell(1)?; let execute_at = cell(2)?; let interval_value = cell(3)?; let interval_field = cell(4)?; let body = cell(5)?; let schedule = if interval_value.is_empty() && interval_field.is_empty() { format!("AT {execute_at}") } else { format!("EVERY {interval_value} {interval_field}") }; Ok(mysql_event_ddl(database, name, status, &schedule, body)) } _ => Err(format!("MySQL object DDL extraction not supported for {:?}", kind)), } } pub fn oracle_object_source_query(kind: &TransferObjectKind, schema: &str, name: &str) -> Result { let object_type = match kind { TransferObjectKind::View => "VIEW", TransferObjectKind::MaterializedView => "MATERIALIZED_VIEW", TransferObjectKind::Procedure => "PROCEDURE", TransferObjectKind::Function => "FUNCTION", TransferObjectKind::Trigger => "TRIGGER", TransferObjectKind::Sequence => "SEQUENCE", _ => return Err(format!("Oracle object source not supported for {:?}", kind)), }; let name_lit = quote_string_literal(name); if schema.trim().is_empty() { Ok(format!("SELECT DBMS_METADATA.GET_DDL({}, {}) FROM DUAL", quote_string_literal(object_type), name_lit)) } else { Ok(format!( "SELECT DBMS_METADATA.GET_DDL({}, {}, {}) FROM DUAL", quote_string_literal(object_type), name_lit, quote_string_literal(schema) )) } } /// Rewrite `"SCHEMA"."NAME"` occurrences in Oracle/DM metadata DDL from /// source_schema to target_schema. pub fn sqlserver_object_source_query(kind: &TransferObjectKind, schema: &str, name: &str) -> Result { match kind { TransferObjectKind::View | TransferObjectKind::Procedure | TransferObjectKind::Function | TransferObjectKind::Trigger => { let object_type = match kind { TransferObjectKind::View => "'V'", TransferObjectKind::Procedure => "'P'", TransferObjectKind::Function => "'FN','IF','TF','FS','FT'", _ => "'TR'", }; Ok(format!( "SELECT m.definition FROM sys.sql_modules m \ JOIN sys.objects o ON o.object_id = m.object_id \ JOIN sys.schemas s ON s.schema_id = o.schema_id \ WHERE s.name = {} AND o.name = {} AND o.type IN ({})", quote_string_literal(schema), quote_string_literal(name), object_type )) } TransferObjectKind::Sequence => Ok(format!( "SELECT CAST(seq.start_value AS NVARCHAR(50)), \ CAST(seq.increment_value AS NVARCHAR(50)), \ CAST(seq.minimum_value AS NVARCHAR(50)), \ CAST(seq.maximum_value AS NVARCHAR(50)), \ CASE WHEN seq.is_cycling = 1 THEN 'CYCLE' ELSE 'NO CYCLE' END, \ CASE WHEN seq.is_cached = 1 THEN CAST(seq.cache_size AS NVARCHAR(50)) ELSE '0' END \ FROM sys.sequences seq JOIN sys.schemas s ON s.schema_id = seq.schema_id \ WHERE s.name = {} AND seq.name = {}", quote_string_literal(schema), quote_string_literal(name) )), _ => Err(format!("SQL Server object source not supported for {:?}", kind)), } } pub fn sqlserver_object_ddl_from_result( result: &db::QueryResult, schema: &str, name: &str, kind: &TransferObjectKind, ) -> Result { let Some(row) = result.rows.first() else { return Err(format!("No DDL returned for SQL Server {kind:?} {name}")); }; match kind { TransferObjectKind::View | TransferObjectKind::Procedure | TransferObjectKind::Function | TransferObjectKind::Trigger => row .first() .and_then(|v| v.as_str()) .map(|s| s.to_string()) .ok_or_else(|| format!("No DDL returned for SQL Server {kind:?} {name}")), TransferObjectKind::Sequence => { let cell = |i: usize| row.get(i).and_then(|v| v.as_str()).unwrap_or(""); let start = cell(0); let increment = cell(1); let minimum = cell(2); let maximum = cell(3); let cycle = cell(4); let cache = cell(5); Ok(format!( "CREATE SEQUENCE [{}].[{}] START WITH {} INCREMENT BY {} MINVALUE {} MAXVALUE {} {} {}", schema, name, start, increment, minimum, maximum, cycle, if cache == "0" { "NO CACHE".to_string() } else { format!("CACHE {cache}") } )) } _ => Err(format!("SQL Server object DDL not supported for {:?}", kind)), } } fn rewrite_double_quoted_schema_qualifier(ddl: &str, source_schema: &str, target_schema: &str) -> String { if source_schema == target_schema || source_schema.is_empty() { return ddl.to_string(); } let source = format!("\"{}\".", source_schema.replace('"', "\"\"")); let target = format!("\"{}\".", target_schema.replace('"', "\"\"")); map_sql_code_spans(ddl, false, |code| code.replace(&source, &target)) } pub fn rewrite_oracle_schema_qualifier(ddl: &str, source_schema: &str, target_schema: &str) -> String { rewrite_double_quoted_schema_qualifier(ddl, source_schema, target_schema) } pub(crate) fn quote_postgres_string_literal(value: &str) -> String { if !value.contains('\\') && !value.chars().any(|character| character.is_ascii_control()) { return quote_string_literal(value); } let mut escaped = String::with_capacity(value.len()); for character in value.chars() { match character { '\\' => escaped.push_str("\\\\"), '\n' => escaped.push_str("\\n"), '\r' => escaped.push_str("\\r"), '\t' => escaped.push_str("\\t"), '\x08' => escaped.push_str("\\b"), '\x0c' => escaped.push_str("\\f"), '\'' => escaped.push_str("''"), character if character.is_ascii_control() => { const HEX: &[u8; 16] = b"0123456789ABCDEF"; let byte = character as u8; escaped.push_str("\\x"); escaped.push(HEX[(byte >> 4) as usize] as char); escaped.push(HEX[(byte & 0x0F) as usize] as char); } character => escaped.push(character), } } // Escape string constants keep control characters out of the physical // script and remain correct regardless of standard_conforming_strings. format!("E'{escaped}'") } fn postgres_schema_exists_sql(schema: &str) -> String { format!("SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = {} LIMIT 1", quote_string_literal(schema)) } fn query_result_has_rows(result: &db::QueryResult) -> bool { !result.rows.is_empty() } fn is_simple_identifier(value: &str) -> bool { let mut chars = value.chars(); let Some(first) = chars.next() else { return false; }; if !(first == '_' || first.is_ascii_alphabetic()) { return false; } chars.all(|ch| ch == '_' || ch.is_ascii_alphanumeric()) } fn is_postgres_compat_transfer(source_db: &DatabaseType, target_db: &DatabaseType) -> bool { is_postgres_transfer_dialect(source_db) && is_postgres_transfer_dialect(target_db) } fn is_postgres_transfer_dialect(db_type: &DatabaseType) -> bool { // KingbaseES supports the PostgreSQL DDL, type, and ON CONFLICT paths used by transfer; // other PG-wire databases stay opt-in until their transfer behavior is verified. matches!(db_type, DatabaseType::Postgres | DatabaseType::Kingbase) } fn transfer_table_needs_inline_postgres_schema_ensure( source_db_type: &DatabaseType, target_db_type: &DatabaseType, ) -> bool { is_postgres_transfer_dialect(target_db_type) && !matches!((source_db_type, target_db_type), (DatabaseType::Postgres, DatabaseType::Postgres)) } fn postgres_integer_bounds(data_type: &str) -> Option<(i128, i128)> { let normalized = data_type.trim().to_ascii_lowercase(); match normalized.split(['(', ' ']).next().unwrap_or("") { "smallint" | "int2" => Some((i128::from(i16::MIN), i128::from(i16::MAX))), "integer" | "int4" => Some((i128::from(i32::MIN), i128::from(i32::MAX))), "bigint" | "int8" => Some((i128::from(i64::MIN), i128::from(i64::MAX))), _ => None, } } fn is_postgres_integer_like_type(data_type: &str) -> bool { postgres_integer_bounds(data_type).is_some() } fn sqlserver_integer_bounds(data_type: &str) -> Option<(i128, i128)> { let normalized = data_type.trim().to_ascii_lowercase(); match normalized.split(['(', ' ']).next().unwrap_or("") { "bit" => Some((i128::MIN, i128::MAX)), "tinyint" => Some((0, i128::from(u8::MAX))), "smallint" => Some((i128::from(i16::MIN), i128::from(i16::MAX))), "int" | "integer" => Some((i128::from(i32::MIN), i128::from(i32::MAX))), "bigint" => Some((i128::from(i64::MIN), i128::from(i64::MAX))), _ => None, } } pub(crate) fn normalize_integer_literal( value: &str, db_type: &DatabaseType, column_type: Option<&str>, ) -> Option { let bounds = match db_type { db_type if is_postgres_transfer_dialect(db_type) => column_type.and_then(postgres_integer_bounds), DatabaseType::SqlServer => column_type.and_then(sqlserver_integer_bounds), _ => None, }?; // Excel numeric cells arrive as f64; normalize only an explicit zero fraction so real decimals, // scientific notation, and values outside the target integer range stay untouched. if value.bytes().any(|byte| matches!(byte, b'e' | b'E')) { return None; } let (integer, fraction) = value.split_once('.')?; if fraction.is_empty() || !fraction.bytes().all(|byte| byte == b'0') { return None; } let digits = integer.strip_prefix('-').or_else(|| integer.strip_prefix('+')).unwrap_or(integer); if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { return None; } let parsed = integer.parse::().ok()?; if parsed < bounds.0 || parsed > bounds.1 { return None; } Some(integer.to_string()) } fn is_postgres_numeric_family(data_type: &str) -> bool { let normalized = data_type.trim().to_ascii_lowercase(); let base = normalized.split(['(', ' ']).next().unwrap_or(""); matches!( base, "smallint" | "int2" | "integer" | "int4" | "bigint" | "int8" | "numeric" | "decimal" | "real" | "float4" | "float" | "double" | "doubleprecision" | "float8" ) } /// Strips validated en-US thousands separators from a numeric literal for numeric target /// columns. Only standard 3-digit grouping is accepted ("1,234", "12,345,678"); malformed /// grouping ("1,23,4", "1,,234") or any non-numeric character returns None so the original /// text reaches the database and keeps its existing validation error instead of being /// silently coerced. Values without a comma are left untouched. pub(crate) fn normalize_thousands_numeric_literal( value: &str, db_type: &DatabaseType, column_type: Option<&str>, ) -> Option { if !is_postgres_transfer_dialect(db_type) || !column_type.is_some_and(is_postgres_numeric_family) { return None; } let trimmed = value.trim(); if trimmed.is_empty() || trimmed.bytes().any(|byte| matches!(byte, b'e' | b'E')) { return None; } let (negative, unsigned) = match trimmed.as_bytes().first() { Some(b'-') => (true, &trimmed[1..]), Some(b'+') => (false, &trimmed[1..]), _ => (false, trimmed), }; if unsigned.is_empty() { return None; } let (integer_part, fraction) = match unsigned.split_once('.') { Some((integer, fraction)) => (integer, Some(fraction)), None => (unsigned, None), }; if fraction.is_some_and(|fraction| fraction.is_empty() || !fraction.bytes().all(|byte| byte.is_ascii_digit())) { return None; } let mut digits = String::with_capacity(unsigned.len()); for (index, group) in integer_part.split(',').enumerate() { if group.is_empty() || !group.bytes().all(|byte| byte.is_ascii_digit()) { return None; } if (index == 0 && group.len() > 3) || (index > 0 && group.len() != 3) { return None; } digits.push_str(group); } if !integer_part.contains(',') { return None; } let mut canonical = String::with_capacity(trimmed.len()); if negative { canonical.push('-'); } canonical.push_str(&digits); if let Some(fraction) = fraction { canonical.push('.'); canonical.push_str(fraction); } Some(canonical) } fn is_postgres_sequence_default(default_value: Option<&str>) -> bool { default_value.is_some_and(|value| value.to_ascii_lowercase().contains("nextval(")) } fn is_postgres_generated_extra(extra: Option<&str>) -> bool { extra.is_some_and(|value| value.trim().to_ascii_lowercase().starts_with("generated ")) } fn is_postgres_identity_extra(extra: Option<&str>) -> bool { extra.is_some_and(|value| { let normalized = value.trim().to_ascii_lowercase(); normalized.starts_with("generated ") && normalized.contains(" identity") }) } fn is_postgres_generated_always_identity_extra(extra: Option<&str>) -> bool { extra.is_some_and(|value| { let mut parts = value.split_whitespace(); parts.next().is_some_and(|part| part.eq_ignore_ascii_case("generated")) && parts.next().is_some_and(|part| part.eq_ignore_ascii_case("always")) && parts.next().is_some_and(|part| part.eq_ignore_ascii_case("as")) && parts.next().is_some_and(|part| part.eq_ignore_ascii_case("identity")) }) } pub(crate) fn is_identity_column_extra(extra: Option<&str>) -> bool { extra.is_some_and(|value| { let normalized = value.trim().to_ascii_lowercase(); normalized.contains("identity") || normalized.contains("auto_increment") || normalized.contains("autoincrement") }) } pub(crate) fn is_mysql_generated_column_extra(extra: Option<&str>) -> bool { extra.is_some_and(|value| { let mut parts = value.split_whitespace(); let Some(first) = parts.next() else { return false; }; if first.eq_ignore_ascii_case("generated") { return true; } matches!(first.to_ascii_lowercase().as_str(), "virtual" | "stored" | "persistent") && parts.next().is_some_and(|part| part.eq_ignore_ascii_case("generated")) }) } #[cfg(test)] fn selected_columns_include_identity_extras(columns: &[String], column_extras: &[Option]) -> bool { columns .iter() .enumerate() .any(|(index, _)| is_identity_column_extra(column_extras.get(index).and_then(|extra| extra.as_deref()))) } fn selected_columns_include_identity_columns(columns: &[String], all_columns: &[db::ColumnInfo]) -> bool { all_columns.iter().any(|column| { is_identity_column_extra(column.extra.as_deref()) && columns.iter().any(|name| name.eq_ignore_ascii_case(&column.name)) }) } fn selected_columns_include_postgres_generated_always_identity_columns( columns: &[String], all_columns: &[db::ColumnInfo], ) -> bool { all_columns.iter().any(|column| { is_postgres_generated_always_identity_extra(column.extra.as_deref()) && columns.iter().any(|name| name.eq_ignore_ascii_case(&column.name)) }) } fn is_sqlserver_rowversion_type(data_type: &str) -> bool { let normalized = data_type.trim().to_ascii_lowercase(); matches!(normalized.split(['(', ' ', '\t', '\n']).next().unwrap_or(""), "timestamp" | "rowversion") } fn is_sqlserver_non_insertable_transfer_column( column: &db::ColumnInfo, source_db_type: &DatabaseType, target_db_type: &DatabaseType, ) -> bool { matches!((source_db_type, target_db_type), (DatabaseType::SqlServer, DatabaseType::SqlServer)) && is_sqlserver_rowversion_type(&column.data_type) } fn is_mysql_non_insertable_transfer_column(column: &db::ColumnInfo, source_db_type: &DatabaseType) -> bool { *source_db_type == DatabaseType::Mysql && is_mysql_generated_column_extra(column.extra.as_deref()) } fn writable_transfer_columns( columns: &[db::ColumnInfo], source_db_type: &DatabaseType, target_db_type: &DatabaseType, ) -> Vec { columns .iter() .filter(|column| { !is_sqlserver_non_insertable_transfer_column(column, source_db_type, target_db_type) && !is_mysql_non_insertable_transfer_column(column, source_db_type) }) .cloned() .collect() } fn transfer_column_names_match( target_db_type: &DatabaseType, quote_target_column_names: bool, left: &str, right: &str, ) -> bool { if matches!( target_db_type, DatabaseType::Mysql | DatabaseType::Goldendb | DatabaseType::Sqlite | DatabaseType::Rqlite | DatabaseType::CloudflareD1 | DatabaseType::DuckDb | DatabaseType::SqlServer | DatabaseType::Doris | DatabaseType::StarRocks | DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala | DatabaseType::Spark | DatabaseType::Access ) || (matches!(target_db_type, DatabaseType::Gaussdb | DatabaseType::OpenGauss) && !quote_target_column_names) { // Unquoted GaussDB/openGauss identifiers fold to lowercase server-side, // so with quoting disabled a table created from mixed-case source // columns reports folded names back from the catalog. left.eq_ignore_ascii_case(right) } else { left == right } } fn missing_transfer_target_columns( target_columns: &[db::ColumnInfo], col_names: &[String], target_db_type: &DatabaseType, quote_target_column_names: bool, ) -> Vec { col_names .iter() .filter(|name| { !target_columns.iter().any(|column| { transfer_column_names_match(target_db_type, quote_target_column_names, name, &column.name) }) }) .cloned() .collect() } fn target_column_can_be_omitted(column: &db::ColumnInfo, target_db_type: &DatabaseType) -> bool { let extra = column.extra.as_deref().unwrap_or_default().trim().to_ascii_lowercase(); column.is_nullable || column.column_default.as_deref().is_some_and(|value| !value.trim().is_empty()) || extra.contains("generated") || extra.contains("identity") || extra.contains("auto_increment") || extra.contains("autoincrement") || extra.contains("computed") || (matches!(target_db_type, DatabaseType::SqlServer) && is_sqlserver_rowversion_type(&column.data_type)) } fn required_unmapped_transfer_target_columns( target_columns: &[db::ColumnInfo], col_names: &[String], target_db_type: &DatabaseType, quote_target_column_names: bool, ) -> Vec { target_columns .iter() .filter(|column| { !target_column_can_be_omitted(column, target_db_type) && !col_names.iter().any(|name| { transfer_column_names_match(target_db_type, quote_target_column_names, name, &column.name) }) }) .map(|column| column.name.clone()) .collect() } /// Fails fast when a preexisting target table's structure can't accept the /// source columns. DBX never alters an existing target table's columns, so /// skipping this check (structure-only transfers used to silently skip it, /// #7660) leaves the transfer reporting success while the target quietly stays /// out of sync with the source. fn validate_preexisting_target_columns( target_columns: &[db::ColumnInfo], col_names: &[String], target_db_type: &DatabaseType, quote_target_column_names: bool, target_table: &str, ) -> Result<(), String> { let missing = missing_transfer_target_columns(target_columns, col_names, target_db_type, quote_target_column_names); if !missing.is_empty() { return Err(format!( "Target table '{target_table}' already exists with a different structure and is missing column(s) \ {} present in the source table. DBX does not alter an existing target table's columns during \ transfer — drop the target table or adjust its structure to match the source first.", missing.join(", ") )); } let required = required_unmapped_transfer_target_columns(target_columns, col_names, target_db_type, quote_target_column_names); if !required.is_empty() { return Err(format!( "Target table '{target_table}' already exists with a different structure and has required column(s) \ {} that are not present in the source table and have no default or generated value. DBX does not \ alter an existing target table's columns during transfer — drop the target table or adjust its \ structure to match the source first.", required.join(", ") )); } Ok(()) } fn transfer_key_columns(columns: &[db::ColumnInfo], db_type: &DatabaseType) -> Vec { let uses_unique_key_model = matches!(db_type, DatabaseType::Doris | DatabaseType::StarRocks); columns .iter() .filter(|column| column.is_primary_key || (uses_unique_key_model && column.is_unique)) .map(|column| column.name.clone()) .collect() } fn identity_insert_statement(table: &str, schema: &str, db_type: &DatabaseType, enabled: bool) -> String { let full_table = qualified_table(table, schema, db_type, None); format!("SET IDENTITY_INSERT {full_table} {}", if enabled { "ON" } else { "OFF" }) } #[cfg(test)] fn wrap_dameng_identity_insert_sql(insert_sql: &str, table: &str, schema: &str) -> String { let full_table = qualified_table(table, schema, &DatabaseType::Dameng, None); wrap_dameng_identity_insert_sql_for_table(insert_sql, &full_table) } pub(crate) fn wrap_dameng_identity_insert_sql_for_table(insert_sql: &str, full_table: &str) -> String { let trimmed = insert_sql.trim().trim_end_matches(';').trim(); format!("SET IDENTITY_INSERT {full_table} ON;\n{trimmed};\nSET IDENTITY_INSERT {full_table} OFF;") } async fn execute_transfer_write_statement( state: &AppState, target_pool_key: &str, sql: &str, target_db_type: &DatabaseType, table: &str, schema: &str, needs_identity_insert: bool, ) -> Result<(), String> { if !needs_identity_insert || !matches!(target_db_type, DatabaseType::Dameng | DatabaseType::SqlServer) { execute_on_pool(state, target_pool_key, sql).await?; return Ok(()); } let enable_sql = identity_insert_statement(table, schema, target_db_type, true); let disable_sql = identity_insert_statement(table, schema, target_db_type, false); execute_on_pool(state, target_pool_key, &enable_sql) .await .map_err(|e| format!("Failed to enable IDENTITY_INSERT for {table}: {e}"))?; let write_result = execute_on_pool(state, target_pool_key, sql).await; let disable_result = execute_on_pool(state, target_pool_key, &disable_sql).await; match (write_result, disable_result) { (Ok(_), Ok(_)) => Ok(()), (Err(write_error), Ok(_)) => Err(write_error), (Ok(_), Err(disable_error)) => Err(format!("Failed to disable IDENTITY_INSERT for {table}: {disable_error}")), (Err(write_error), Err(disable_error)) => { Err(format!("{write_error}; also failed to disable IDENTITY_INSERT for {table}: {disable_error}")) } } } fn rewrite_postgres_schema_qualified_references(input: &str, source_schema: &str, target_schema: &str) -> String { if source_schema.trim().is_empty() || source_schema == target_schema { return input.to_string(); } let quoted_source = format!("{}.", quote_identifier(source_schema, &DatabaseType::Postgres)); let quoted_target = format!("{}.", quote_identifier(target_schema, &DatabaseType::Postgres)); let rewritten = input.replace("ed_source, "ed_target); let unquoted_pattern = Regex::new(&format!(r#"(^|[^"\w]){}\."#, regex::escape(source_schema))).expect("valid postgres schema regex"); unquoted_pattern .replace_all(&rewritten, |captures: ®ex::Captures| format!("{}{}", &captures[1], quoted_target)) .into_owned() } fn postgres_column_type_sql( column: &db::ColumnInfo, source_schema: &str, target_schema: &str, source_db: &DatabaseType, target_db: &DatabaseType, ) -> String { if let Some(mapped_type) = clickhouse_temporal_column_type(column, source_db, target_db) { return mapped_type; } if is_postgres_compat_transfer(source_db, target_db) { let trimmed = column.data_type.trim(); if !trimmed.is_empty() { return rewrite_postgres_schema_qualified_references(trimmed, source_schema, target_schema); } } map_column_type(&column.data_type, source_db, target_db) } fn clickhouse_temporal_column_type( column: &db::ColumnInfo, source_db: &DatabaseType, target_db: &DatabaseType, ) -> Option { if !matches!(target_db, DatabaseType::ClickHouse) || source_db == target_db { return None; } let source_type = column.data_type.trim(); let lower = source_type.to_ascii_lowercase(); let base = lower.split(['(', ' ', '\t', '\n']).next().unwrap_or("").trim(); if !matches!(base, "datetime" | "timestamp" | "timestamptz") { return None; } let scale = clickhouse_datetime64_scale(column); // ClickHouse DateTime stores whole seconds, and older versions reject // fractional timestamp strings such as Dameng's TIMESTAMP(6) output. Some(format!("DateTime64({scale})")) } fn clickhouse_datetime64_scale(column: &db::ColumnInfo) -> u8 { let scale = parse_temporal_type_scale(&column.data_type).or(column.numeric_scale).unwrap_or(6); scale.clamp(0, 9) as u8 } fn parse_temporal_type_scale(source_type: &str) -> Option { let start = source_type.find('(')? + 1; let rest = &source_type[start..]; let digits = rest.bytes().take_while(|byte| byte.is_ascii_digit()).collect::>(); if digits.is_empty() { return None; } std::str::from_utf8(&digits).ok()?.parse::().ok() } fn postgres_default_clause( column: &db::ColumnInfo, source_schema: &str, target_schema: &str, source_db: &DatabaseType, target_db: &DatabaseType, ) -> Option { if !is_postgres_compat_transfer(source_db, target_db) { return None; } if is_postgres_generated_extra(column.extra.as_deref()) { if let Some(extra) = column.extra.as_deref().map(str::trim).filter(|value| !value.is_empty()) { return Some(extra.to_string()); } } let default_value = column.column_default.as_deref()?.trim(); if default_value.is_empty() { return None; } if is_postgres_sequence_default(Some(default_value)) && is_postgres_integer_like_type(&column.data_type) { return Some("GENERATED BY DEFAULT AS IDENTITY".to_string()); } Some(format!( "DEFAULT {}", rewrite_postgres_schema_qualified_references(default_value, source_schema, target_schema) )) } fn is_mysql_family_target(target_db: &DatabaseType) -> bool { matches!( target_db, DatabaseType::Mysql | DatabaseType::Doris | DatabaseType::StarRocks | DatabaseType::Goldendb | DatabaseType::Sundb ) } fn supports_deferred_mysql_foreign_keys(target_db: &DatabaseType) -> bool { is_mysql_family_target(target_db) && crate::table_structure_sql::supports_foreign_keys(*target_db) } /// QuestDB is not included. It only uses the PGWire protocol. SQL DDL syntax is not compatible. fn is_postgres_family_target(target_db: &DatabaseType) -> bool { matches!( target_db, DatabaseType::Postgres | DatabaseType::Gaussdb | DatabaseType::OpenGauss | DatabaseType::Redshift | DatabaseType::Kingbase | DatabaseType::Highgo | DatabaseType::Uxdb | DatabaseType::Kwdb | DatabaseType::Vastbase ) } fn is_mysql_numeric_base_type(data_type: &str) -> bool { let normalized = data_type.trim().to_ascii_lowercase(); let base = normalized.split(['(', ' ']).next().unwrap_or(""); matches!( base, "tinyint" | "smallint" | "mediumint" | "int" | "integer" | "bigint" | "decimal" | "numeric" | "float" | "double" | "real" | "bit" | "year" ) } fn is_mysql_function_default(raw: &str) -> bool { let trimmed = raw.trim(); if trimmed.eq_ignore_ascii_case("NULL") { return true; } let upper = trimmed.to_ascii_uppercase(); if upper == "CURRENT_TIMESTAMP" || upper.starts_with("CURRENT_TIMESTAMP(") { return true; } if upper == "LOCALTIME" || upper.starts_with("LOCALTIME(") { return true; } if upper == "LOCALTIMESTAMP" || upper.starts_with("LOCALTIMESTAMP(") { return true; } matches!(upper.as_str(), "CURRENT_DATE" | "CURRENT_TIME" | "NOW()" | "UTC_TIMESTAMP()" | "UUID()") } fn looks_like_numeric_literal(raw: &str) -> bool { let trimmed = raw.trim(); if trimmed.is_empty() { return false; } trimmed.parse::().is_ok() || trimmed.parse::().is_ok() || trimmed.parse::().is_ok_and(|value| value.is_finite()) } fn format_mysql_default_literal(raw: &str, data_type: &str) -> String { let trimmed = raw.trim(); if trimmed.eq_ignore_ascii_case("NULL") { return "NULL".to_string(); } if is_mysql_function_default(trimmed) { return trimmed.to_string(); } if trimmed.len() >= 2 && trimmed.starts_with('\'') && trimmed.ends_with('\'') { return trimmed.to_string(); } if is_mysql_numeric_base_type(data_type) && looks_like_numeric_literal(trimmed) { return trimmed.to_string(); } format!("'{}'", trimmed.replace('\'', "''")) } fn column_default_clause( column: &db::ColumnInfo, source_schema: &str, target_schema: &str, source_db: &DatabaseType, target_db: &DatabaseType, ) -> Option { if is_postgres_compat_transfer(source_db, target_db) { return postgres_default_clause(column, source_schema, target_schema, source_db, target_db); } if is_mysql_family_target(target_db) { let default_value = column.column_default.as_deref()?.trim(); if default_value.is_empty() { return None; } return Some(format!("DEFAULT {}", format_mysql_default_literal(default_value, &column.data_type))); } None } #[derive(Debug, Default, PartialEq, Eq)] struct MysqlExtraClauses { auto_increment: bool, on_update: Option, } fn parse_mysql_extra_clauses(extra: Option<&str>) -> MysqlExtraClauses { let mut result = MysqlExtraClauses::default(); let Some(raw) = extra else { return result; }; let trimmed = raw.trim(); if trimmed.is_empty() { return result; } let lowered = trimmed.to_ascii_lowercase(); if lowered.contains("auto_increment") { result.auto_increment = true; } let pattern = Regex::new(r"(?i)\bon\s+update\s+(.+)$").expect("valid mysql on-update regex"); if let Some(captures) = pattern.captures(trimmed) { let raw_expr = captures.get(1).map(|m| m.as_str()).unwrap_or(""); let cleaned = raw_expr.trim().trim_end_matches([',', ';', ' ']).trim(); if !cleaned.is_empty() { result.on_update = Some(cleaned.to_string()); } } result } fn postgres_order_by_expression(columns: &[String], db_type: &DatabaseType) -> Option { postgres_order_by_expression_with_identifier_quote(columns, db_type, None) } fn postgres_order_by_expression_with_identifier_quote( columns: &[String], db_type: &DatabaseType, identifier_quote: Option<&str>, ) -> Option { if columns.is_empty() { return None; } Some( columns .iter() .map(|column| quote_identifier_with_identifier_quote(column, db_type, identifier_quote)) .collect::>() .join(", "), ) } fn oracle_rownum_page_sql(col_list: &str, base_sql: String, offset: u64, limit: usize) -> String { if offset == 0 { return format!("SELECT {col_list} FROM ({base_sql}) WHERE ROWNUM <= {limit}"); } let end = offset + limit as u64; format!( "SELECT {col_list} FROM (SELECT dbx_inner.*, ROWNUM AS \"__dbx_row_num\" FROM ({base_sql}) dbx_inner WHERE ROWNUM <= {end}) WHERE \"__dbx_row_num\" > {offset}" ) } // SQL Server 2008 R2 and older reject `OFFSET ... FETCH` (added in SQL Server // 2012), so paged reads must use a ROW_NUMBER() subquery that every supported // SQL Server version accepts (issue #7356). fn sqlserver_row_number_page_sql( col_list: &str, from_clause: &str, order_by: &str, offset: u64, limit: usize, ) -> String { let end = offset + limit as u64; format!( "SELECT {col_list} FROM (SELECT {col_list}, ROW_NUMBER() OVER (ORDER BY {order_by}) AS __dbx_row_num FROM {from_clause}) AS __dbx_page WHERE __dbx_row_num > {offset} AND __dbx_row_num <= {end}" ) } fn postgres_index_column_sql(column: &str) -> String { if is_simple_identifier(column) { quote_identifier(column, &DatabaseType::Postgres) } else { column.to_string() } } fn generate_postgres_index_ddl(indexes: &[db::IndexInfo], table: &str, schema: &str) -> Vec { let full_table = qualified_table(table, schema, &DatabaseType::Postgres, None); let mut statements = Vec::new(); for index in indexes.iter().filter(|index| !index.is_primary) { if index.name.trim().is_empty() || index.columns.is_empty() { continue; } let unique = if index.is_unique { "UNIQUE " } else { "" }; let using_clause = index .index_type .as_deref() .map(str::trim) .filter(|value| !value.is_empty()) .map(|value| format!(" USING {value}")) .unwrap_or_default(); let columns = index.columns.iter().map(|column| postgres_index_column_sql(column)).collect::>().join(", "); let include_clause = index .included_columns .as_ref() .filter(|columns| !columns.is_empty()) .map(|columns| { format!( " INCLUDE ({})", columns .iter() .map(|column| quote_identifier(column, &DatabaseType::Postgres)) .collect::>() .join(", ") ) }) .unwrap_or_default(); let filter_clause = index .filter .as_deref() .map(str::trim) .filter(|value| !value.is_empty()) .map(|value| format!(" WHERE {value}")) .unwrap_or_default(); statements.push(format!( "CREATE {unique}INDEX IF NOT EXISTS {} ON {full_table}{using_clause} ({columns}){include_clause}{filter_clause}", quote_identifier(&index.name, &DatabaseType::Postgres) )); if let Some(comment) = index.comment.as_deref().map(str::trim).filter(|value| !value.is_empty()) { let qualified_index = if schema.is_empty() { quote_identifier(&index.name, &DatabaseType::Postgres) } else { format!( "{}.{}", quote_identifier(schema, &DatabaseType::Postgres), quote_identifier(&index.name, &DatabaseType::Postgres) ) }; statements.push(format!("COMMENT ON INDEX {qualified_index} IS {}", quote_string_literal(comment))); } } statements } /// Groups foreign keys by constraint name, preserving first-seen order — MySQL /// and Postgres both report one row per (constraint, column) pair for /// multi-column foreign keys, so callers need the columns regrouped by /// constraint before they can emit one `ADD CONSTRAINT` statement per key. fn group_foreign_keys_by_constraint_name(foreign_keys: &[db::ForeignKeyInfo]) -> Vec<(&str, Vec<&db::ForeignKeyInfo>)> { let mut grouped: HashMap<&str, Vec<&db::ForeignKeyInfo>> = HashMap::new(); let mut order: Vec<&str> = Vec::new(); for foreign_key in foreign_keys { if !grouped.contains_key(foreign_key.name.as_str()) { order.push(foreign_key.name.as_str()); } grouped.entry(foreign_key.name.as_str()).or_default().push(foreign_key); } order.into_iter().filter_map(|name| grouped.remove(name).map(|group| (name, group))).collect() } fn generate_postgres_foreign_key_ddl( foreign_keys: &[db::ForeignKeyInfo], table: &str, source_schema: &str, target_schema: &str, ) -> Vec { let full_table = qualified_table(table, target_schema, &DatabaseType::Postgres, None); let mut statements = Vec::new(); for (name, group) in group_foreign_keys_by_constraint_name(foreign_keys) { let columns = group .iter() .map(|foreign_key| quote_identifier(&foreign_key.column, &DatabaseType::Postgres)) .collect::>() .join(", "); let ref_columns = group .iter() .map(|foreign_key| quote_identifier(&foreign_key.ref_column, &DatabaseType::Postgres)) .collect::>() .join(", "); let referenced_schema = match group[0].ref_schema.as_deref() { Some(ref_schema) if ref_schema == source_schema => target_schema, Some(ref_schema) => ref_schema, None => target_schema, }; let referenced_table = qualified_table(&group[0].ref_table, referenced_schema, &DatabaseType::Postgres, None); statements.push(format!( "ALTER TABLE {full_table} ADD CONSTRAINT {} FOREIGN KEY ({columns}) REFERENCES {referenced_table} ({ref_columns})", quote_identifier(name, &DatabaseType::Postgres) )); } statements } async fn restore_postgres_table_schema_objects( state: &AppState, target_pool_key: &str, target_table: &str, source_schema: &str, target_schema: &str, source_indexes: &[db::IndexInfo], source_foreign_keys: &[db::ForeignKeyInfo], ) -> Result<(), String> { for statement in generate_postgres_index_ddl(source_indexes, target_table, target_schema) { execute_on_pool(state, target_pool_key, &statement) .await .map_err(|e| format!("Failed to create PostgreSQL index for {target_table}: {e}"))?; } for statement in generate_postgres_foreign_key_ddl(source_foreign_keys, target_table, source_schema, target_schema) { execute_on_pool(state, target_pool_key, &statement) .await .map_err(|e| format!("Failed to create PostgreSQL foreign key for {target_table}: {e}"))?; } Ok(()) } /// Builds deferred `ALTER TABLE ... ADD CONSTRAINT ... FOREIGN KEY` statements for a /// MySQL-family target table from structured source foreign key metadata. /// /// Used instead of inline `CREATE TABLE ... FOREIGN KEY` so table creation order /// never has to satisfy foreign key dependencies — this is what makes transferring /// tables with a foreign key cycle (or any dependency the sort couldn't fully /// resolve) possible at all, mirroring the existing Postgres transfer path. fn generate_mysql_foreign_key_alter_statements( foreign_keys: &[db::ForeignKeyInfo], request: &TransferRequest, target_table: &str, target_db_type: &DatabaseType, ) -> Vec { // MySQL has no separate "schema" concept — `database` doubles as the schema, // and callers that leave `source_schema` empty (the common case for MySQL // transfers) still need something to compare `ForeignKeyInfo.ref_schema` // against. Mirrors `mysql_table_metadata_catalog`'s schema-or-database // fallback (crates/dbx-core/src/schema.rs), which is private to that module. let source_database = if request.source_schema.trim().is_empty() { request.source_database.as_str() } else { request.source_schema.as_str() }; let full_table = quote_identifier(target_table, target_db_type); let mut statements = Vec::new(); for (name, group) in group_foreign_keys_by_constraint_name(foreign_keys) { let columns = group .iter() .map(|foreign_key| quote_identifier(&foreign_key.column, target_db_type)) .collect::>() .join(", "); let ref_columns = group .iter() .map(|foreign_key| quote_identifier(&foreign_key.ref_column, target_db_type)) .collect::>() .join(", "); let referenced_table = match group[0].ref_schema.as_deref() { // Referenced table lives in the same database this transfer is // reading from, so it's part of (or expected to be part of) this // transfer batch — resolve its target-side name the same way every // other transferred table's name is resolved (case rules, etc.). Some(ref_schema) if ref_schema == source_database => { quote_identifier(&request.target_table_name(&group[0].ref_table), target_db_type) } // Genuine cross-database foreign key pointing outside the transfer's // selected tables: that table was never created or renamed by this // transfer, so reference it by its original database/name, assumed // to already exist unchanged on the target server. Some(ref_schema) => { format!( "{}.{}", quote_identifier(ref_schema, target_db_type), quote_identifier(&group[0].ref_table, target_db_type) ) } None => quote_identifier(&request.target_table_name(&group[0].ref_table), target_db_type), }; let mut statement = format!( "ALTER TABLE {full_table} ADD CONSTRAINT {} FOREIGN KEY ({columns}) REFERENCES {referenced_table} ({ref_columns})", quote_identifier(name, target_db_type) ); if let Some(on_delete) = group[0].on_delete.as_deref() { statement.push_str(&format!(" ON DELETE {on_delete}")); } if let Some(on_update) = group[0].on_update.as_deref() { statement.push_str(&format!(" ON UPDATE {on_update}")); } statements.push(statement); } statements } fn generate_postgres_sequence_sync_sql(columns: &[db::ColumnInfo], table: &str, schema: &str) -> Vec { let full_table = qualified_table(table, schema, &DatabaseType::Postgres, None); columns .iter() .filter(|column| { is_postgres_sequence_default(column.column_default.as_deref()) || is_postgres_identity_extra(column.extra.as_deref()) }) .map(|column| { let quoted_column = quote_identifier(&column.name, &DatabaseType::Postgres); format!( "SELECT setval(pg_get_serial_sequence({}, {})::regclass, GREATEST(COALESCE(MAX({quoted_column}::bigint), 0), 1), MAX({quoted_column}::bigint) IS NOT NULL) FROM {full_table}", quote_string_literal(&full_table), quote_string_literal(&column.name) ) }) .collect() } #[derive(Debug, Clone)] struct PostgresOwnedSequence { name: String, owner_table: String, owner_column: String, } #[derive(Debug, Clone, PartialEq, Eq)] struct PostgresSequenceSnapshot { name: String, owner_table: Option, owner_column: Option, } #[derive(Debug, Clone, PartialEq, Eq)] struct PostgresTransferSequence { name: String, data_type: String, start_value: String, min_value: String, max_value: String, increment: String, cycle: bool, cache_value: String, last_value: Option, } fn postgres_sequence_qualified_name(schema: &str, sequence_name: &str) -> String { if schema.trim().is_empty() { quote_identifier(sequence_name, &DatabaseType::Postgres) } else { format!( "{}.{}", quote_identifier(schema, &DatabaseType::Postgres), quote_identifier(sequence_name, &DatabaseType::Postgres) ) } } fn generate_postgres_transfer_sequence_create_ddl(sequence: &PostgresTransferSequence, schema: &str) -> String { let qualified_name = postgres_sequence_qualified_name(schema, &sequence.name); let cycle = if sequence.cycle { "CYCLE" } else { "NO CYCLE" }; format!( "CREATE SEQUENCE IF NOT EXISTS {qualified_name}\n AS {data_type}\n START WITH {start_value}\n INCREMENT BY {increment}\n MINVALUE {min_value}\n MAXVALUE {max_value}\n CACHE {cache_value}\n {cycle}", data_type = sequence.data_type, start_value = sequence.start_value, increment = sequence.increment, min_value = sequence.min_value, max_value = sequence.max_value, cache_value = sequence.cache_value, ) } fn generate_postgres_transfer_sequence_setval_sql(sequence: &PostgresTransferSequence, schema: &str) -> Option { let last_value = sequence.last_value.as_deref()?.trim(); if last_value.is_empty() { return None; } Some(format!( "SELECT setval({}, {last_value}, true)", quote_postgres_string_literal(&postgres_sequence_qualified_name(schema, &sequence.name)) )) } /// Reuse an existing target sequence only when it is already bound to the same /// target table column; otherwise the later `OWNED BY` rebind would silently /// change unrelated objects. fn validate_existing_postgres_sequence( sequence: &PostgresOwnedSequence, existing: Option<&PostgresSequenceSnapshot>, schema: &str, ) -> Result { let Some(existing) = existing else { return Ok(true); }; let owner_matches = match (existing.owner_table.as_deref(), existing.owner_column.as_deref()) { (None, None) => true, (Some(owner_table), Some(owner_column)) => { owner_table == sequence.owner_table && owner_column == sequence.owner_column } _ => false, }; if owner_matches { return Ok(false); } Err(format!( "PostgreSQL sequence {} already exists with incompatible ownership", postgres_sequence_qualified_name(schema, &sequence.name) )) } #[derive(Debug, Clone)] struct PostgresTriggerSource { table_name: String, trigger_name: String, source: String, } #[derive(Debug, Clone)] struct PostgresExtensionSource { extension_name: String, } #[derive(Debug, Clone)] struct PostgresEnumSource { type_name: String, labels: Vec, } #[derive(Debug, Clone)] struct PostgresDomainSource { domain_name: String, base_type: String, default_value: Option, not_null: bool, checks: Vec, } #[derive(Debug, Clone)] struct PostgresMaterializedViewSource { view_name: String, source: String, } #[derive(Debug, Clone)] struct PostgresOwnershipStatement { sql_prefix: String, owner: String, } fn json_string_cell(row: &[serde_json::Value], index: usize) -> Option { row.get(index).and_then(|value| value.as_str().map(str::to_string)) } fn result_rows_to_string_statements(rows: Vec>) -> Vec { rows.into_iter().filter_map(|row| json_string_cell(&row, 0)).filter(|stmt| !stmt.trim().is_empty()).collect() } fn result_rows_to_postgres_ownership_statements(rows: Vec>) -> Vec { rows.into_iter() .filter_map(|row| { let sql_prefix = json_string_cell(&row, 0)?; let owner = json_string_cell(&row, 1)?; if sql_prefix.trim().is_empty() || owner.trim().is_empty() { None } else { Some(PostgresOwnershipStatement { sql_prefix, owner }) } }) .collect() } fn ensure_sql_statement_terminated(sql: &str) -> String { let trimmed = sql.trim(); if trimmed.ends_with(';') { trimmed.to_string() } else { format!("{trimmed};") } } fn generate_postgres_extension_ddl(extension: &PostgresExtensionSource, target_schema: &str) -> String { format!( "CREATE EXTENSION IF NOT EXISTS {} WITH SCHEMA {}", quote_identifier(&extension.extension_name, &DatabaseType::Postgres), quote_identifier(target_schema, &DatabaseType::Postgres) ) } fn generate_postgres_enum_ddl(enum_type: &PostgresEnumSource, target_schema: &str) -> String { let labels = enum_type.labels.iter().map(|label| quote_string_literal(label)).collect::>().join(", "); let create_sql = format!( "CREATE TYPE {}.{} AS ENUM ({labels})", quote_identifier(target_schema, &DatabaseType::Postgres), quote_identifier(&enum_type.type_name, &DatabaseType::Postgres) ); format!( "DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace WHERE n.nspname = {} AND t.typname = {}) THEN {create_sql}; END IF; END $$", quote_string_literal(target_schema), quote_string_literal(&enum_type.type_name) ) } fn generate_postgres_domain_ddl(domain: &PostgresDomainSource, target_schema: &str) -> String { let mut create_sql = format!( "CREATE DOMAIN {}.{} AS {}", quote_identifier(target_schema, &DatabaseType::Postgres), quote_identifier(&domain.domain_name, &DatabaseType::Postgres), domain.base_type ); if let Some(default_value) = domain.default_value.as_deref().map(str::trim).filter(|value| !value.is_empty()) { create_sql.push_str(&format!(" DEFAULT {default_value}")); } if domain.not_null { create_sql.push_str(" NOT NULL"); } for check in &domain.checks { create_sql.push(' '); create_sql.push_str(check); } format!( "DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace WHERE n.nspname = {} AND t.typname = {}) THEN {}; END IF; END $$", quote_string_literal(target_schema), quote_string_literal(&domain.domain_name), create_sql ) } fn generate_postgres_materialized_view_ddls(view: &PostgresMaterializedViewSource, target_schema: &str) -> Vec { let qualified_name = qualified_table(&view.view_name, target_schema, &DatabaseType::Postgres, None); vec![ format!("DROP MATERIALIZED VIEW IF EXISTS {qualified_name}"), format!("CREATE MATERIALIZED VIEW {qualified_name} AS\n{}", ensure_sql_statement_terminated(&view.source)), ] } fn rewrite_postgres_routine_schema(source: &str, source_schema: &str, target_schema: &str) -> Option { let re = Regex::new( r#"(?is)^(\s*CREATE\s+(?:OR\s+REPLACE\s+)?(?:(?:NON)?EDITIONABLE\s+)?(?:FUNCTION|PROCEDURE)\s+)((?:"(?:""|[^"])+"|[A-Za-z_][\w$]*)(?:\s*\.\s*(?:"(?:""|[^"])+"|[A-Za-z_][\w$]*))?)"#, ) .ok()?; let captures = re.captures(source)?; let full = captures.get(0)?; let prefix = captures.get(1)?.as_str(); let existing_name = captures.get(2)?.as_str(); let name_re = Regex::new(r#""(?:""|[^"])+"|[A-Za-z_][\w$]*"#).ok()?; let parts = name_re .find_iter(existing_name) .map(|part| part.as_str().trim().trim_matches('"').replace("\"\"", "\"")) .collect::>(); let name = parts.last()?; let replacement = format!( "{}.{}", quote_identifier(target_schema, &DatabaseType::Postgres), quote_identifier(name, &DatabaseType::Postgres) ); let rewritten = format!("{}{}{}{}", &source[..full.start()], prefix, replacement, &source[full.end()..]); Some(rewrite_postgres_schema_qualified_references(&rewritten, source_schema, target_schema)) } fn rewrite_postgres_trigger_table_schema( source: &str, source_schema: &str, table_name: &str, target_schema: &str, ) -> String { let qualified_target_table = qualified_table(table_name, target_schema, &DatabaseType::Postgres, None); let candidate_patterns = [ format!( " ON {}.{} ", quote_identifier(source_schema, &DatabaseType::Postgres), quote_identifier(table_name, &DatabaseType::Postgres) ), format!(" ON {source_schema}.{table_name} "), format!(" ON {} ", quote_identifier(table_name, &DatabaseType::Postgres)), format!(" ON {table_name} "), ]; for pattern in candidate_patterns { if source.contains(&pattern) { let rewritten = source.replacen(&pattern, &format!(" ON {qualified_target_table} "), 1); return rewrite_postgres_schema_qualified_references(&rewritten, source_schema, target_schema); } } rewrite_postgres_schema_qualified_references(source, source_schema, target_schema) } pub fn escape_value(val: &serde_json::Value, db_type: &DatabaseType) -> String { escape_value_typed(val, db_type, None) } pub fn escape_value_typed(val: &serde_json::Value, db_type: &DatabaseType, column_type: Option<&str>) -> String { match val { serde_json::Value::Null => "NULL".to_string(), serde_json::Value::Bool(b) => match db_type { DatabaseType::Mysql | DatabaseType::Sqlite | DatabaseType::CloudflareD1 | DatabaseType::DuckDb | DatabaseType::Doris | DatabaseType::StarRocks => { if *b { if column_type.is_some_and(is_mysql_bit_type) { "b'1'".to_string() } else { "1".to_string() } } else if column_type.is_some_and(is_mysql_bit_type) { "b'0'".to_string() } else { "0".to_string() } } DatabaseType::SqlServer | DatabaseType::Dameng => { if *b { "1".to_string() } else { "0".to_string() } } _ => { if *b { "TRUE".to_string() } else { "FALSE".to_string() } } }, serde_json::Value::Number(n) => { if let Some(integer_literal) = normalize_integer_literal(&n.to_string(), db_type, column_type) { return integer_literal; } match db_type { DatabaseType::Mysql | DatabaseType::Doris | DatabaseType::StarRocks => { if column_type.is_some_and(is_mysql_bit_type) { format!("b'{}'", n) } else { n.to_string() } } _ => n.to_string(), } } serde_json::Value::String(s) => { if let Some(integer_literal) = normalize_integer_literal(s, db_type, column_type) { return integer_literal; } if let Some(binary_literal) = format_postgres_binary_sql_literal(s, db_type, column_type) { return binary_literal; } if let Some(binary_literal) = format_mysql_binary_sql_literal(s, db_type, column_type) { return binary_literal; } if let Some(binary_literal) = format_sqlserver_binary_sql_literal(s, db_type, column_type) { return binary_literal; } if let Some(numeric_literal) = format_mysql_numeric_string_literal(s, db_type, column_type) { return numeric_literal; } if let Some(temporal_literal) = format_oracle_temporal_sql_literal(s, db_type, column_type) { return temporal_literal; } let literal = format_literal_string(s, db_type, column_type); if *db_type == DatabaseType::Postgres { return quote_postgres_string_literal(&literal); } let escaped = if is_postgres_family_target(db_type) || *db_type == DatabaseType::SqlServer { literal.replace('\'', "''") } else { literal.replace('\\', "\\\\").replace('\'', "''") }; match db_type { DatabaseType::Mysql | DatabaseType::Doris | DatabaseType::StarRocks if column_type.is_some_and(is_mysql_bit_type) => { format!("b'{escaped}'") } DatabaseType::SqlServer => format!("N'{escaped}'"), _ => format!("'{escaped}'"), } } serde_json::Value::Array(arr) => { if *db_type == DatabaseType::Postgres && is_postgres_vector_type(column_type) { return format_postgres_vector_sql_literal(val); } match db_type { DatabaseType::ClickHouse | DatabaseType::Databend => format_ch_array_sql_literal(arr), _ => format_pg_array_sql_literal(arr), } } _ => { let s = val.to_string(); format!("'{}'", s.replace('\\', "\\\\").replace('\'', "''")) } } } fn is_mysql_bit_type(column_type: &str) -> bool { let trimmed = column_type.trim(); let lower = trimmed.to_ascii_lowercase(); lower == "bit" || lower.starts_with("bit(") || lower.starts_with("bit ") } fn is_mysql_numeric_string_literal_database(db_type: &DatabaseType) -> bool { matches!( db_type, DatabaseType::Mysql | DatabaseType::Doris | DatabaseType::StarRocks | DatabaseType::Goldendb | DatabaseType::Sundb ) } fn is_mysql_non_bit_numeric_type(column_type: &str) -> bool { is_mysql_numeric_base_type(column_type) && !is_mysql_bit_type(column_type) } fn format_mysql_numeric_string_literal( value: &str, db_type: &DatabaseType, column_type: Option<&str>, ) -> Option { if !is_mysql_numeric_string_literal_database(db_type) || !column_type.is_some_and(is_mysql_non_bit_numeric_type) { return None; } let trimmed = value.trim(); if looks_like_numeric_literal(trimmed) { Some(trimmed.to_string()) } else { None } } fn is_binary_transfer_column_type(column_type: &str) -> bool { let lower = column_type.trim().to_ascii_lowercase(); let base = lower.split(['(', ' ', '\t', '\n']).next().unwrap_or(""); matches!(base, "binary" | "varbinary" | "blob" | "tinyblob" | "mediumblob" | "longblob" | "bytea" | "image") } fn format_postgres_binary_sql_literal( value: &str, db_type: &DatabaseType, column_type: Option<&str>, ) -> Option { if !matches!(db_type, DatabaseType::Postgres) || !column_type.is_some_and(is_binary_transfer_column_type) { return None; } let hex = value.strip_prefix("0x").or_else(|| value.strip_prefix("0X"))?; if hex.len() % 2 != 0 || !hex.as_bytes().iter().all(|byte| byte.is_ascii_hexdigit()) { return None; } Some(format!("decode('{hex}', 'hex')")) } fn format_mysql_binary_sql_literal(value: &str, db_type: &DatabaseType, column_type: Option<&str>) -> Option { if !matches!(db_type, DatabaseType::Mysql) || !column_type.is_some_and(is_binary_transfer_column_type) { return None; } let trimmed = value.trim(); let hex = trimmed.strip_prefix("0x").or_else(|| trimmed.strip_prefix("0X"))?; if hex.as_bytes().iter().all(|byte| byte.is_ascii_hexdigit()) { Some(if hex.is_empty() { "X''".to_string() } else { format!("0x{hex}") }) } else { None } } fn format_sqlserver_binary_sql_literal( value: &str, db_type: &DatabaseType, column_type: Option<&str>, ) -> Option { if !matches!(db_type, DatabaseType::SqlServer) { return None; } let column_type = column_type.filter(|column_type| is_binary_transfer_column_type(column_type))?; let trimmed = value.trim(); if let Some(hex) = trimmed.strip_prefix("0x").or_else(|| trimmed.strip_prefix("0X")) { if hex.len() % 2 == 0 && hex.as_bytes().iter().all(|byte| byte.is_ascii_hexdigit()) { return Some(format!("0x{hex}")); } } // SQL Server does not implicitly convert NVARCHAR literals to binary targets. // Use the target type so text fallback preserves the same Unicode byte encoding // that a direct typed conversion would produce. let escaped = value.replace('\'', "''"); Some(format!("CONVERT({column_type}, N'{escaped}')")) } fn format_oracle_temporal_sql_literal( value: &str, db_type: &DatabaseType, column_type: Option<&str>, ) -> Option { if !matches!(db_type, DatabaseType::Oracle | DatabaseType::OceanbaseOracle) { return None; } let kind = temporal_column_kind(column_type)?; let normalized_column_type = column_type?.trim().to_ascii_lowercase(); let parts = oracle_export_date_parts(value)?; match kind { "date" => Some(format_oracle_date_sql_literal_parts(&parts)), "datetime" if (normalized_column_type.contains("with time zone") || normalized_column_type.contains("with local time zone")) && parts.zone.is_some() => { let fraction = parts.fraction.unwrap_or_default(); let mask = if fraction.is_empty() { "YYYY-MM-DD HH24:MI:SS" } else { "YYYY-MM-DD HH24:MI:SS.FF" }; let zone = match parts.zone.unwrap_or_default() { "Z" | "z" => "+00:00", zone => zone, }; Some(format!("TO_TIMESTAMP_TZ('{} {}{fraction} {zone}', '{mask} TZH:TZM')", parts.date, parts.time)) } "datetime" => { let fraction = parts.fraction.unwrap_or_default(); let mask = if fraction.is_empty() { "YYYY-MM-DD HH24:MI:SS" } else { "YYYY-MM-DD HH24:MI:SS.FF" }; Some(format!("TO_TIMESTAMP('{} {}{fraction}', '{mask}')", parts.date, parts.time)) } _ => None, } } struct OracleExportDateParts<'a> { date: &'a str, time: &'a str, fraction: Option<&'a str>, zone: Option<&'a str>, } fn format_oracle_date_sql_literal_parts(parts: &OracleExportDateParts<'_>) -> String { if oracle_export_date_parts_are_midnight(parts) { format!("DATE '{}'", parts.date) } else { format!("TO_DATE('{} {}', 'YYYY-MM-DD HH24:MI:SS')", parts.date, parts.time) } } fn oracle_export_date_parts_are_midnight(parts: &OracleExportDateParts<'_>) -> bool { parts.time == "00:00:00" && parts.fraction.map(|fraction| fraction.trim_start_matches('.').chars().all(|ch| ch == '0')).unwrap_or(true) } fn oracle_export_date_parts(value: &str) -> Option> { let bytes = value.as_bytes(); if bytes.len() < 10 || bytes.get(4) != Some(&b'-') || bytes.get(7) != Some(&b'-') { return None; } let date = &value[..10]; if !date.as_bytes().iter().enumerate().all(|(index, byte)| matches!(index, 4 | 7) || byte.is_ascii_digit()) { return None; } if bytes.len() == 10 { return Some(OracleExportDateParts { date, time: "00:00:00", fraction: None, zone: None }); } let separator = *bytes.get(10)?; if separator != b'T' && separator != b' ' { return None; } if bytes.len() < 19 || bytes.get(13) != Some(&b':') || bytes.get(16) != Some(&b':') { return None; } let time = &value[11..19]; if !time.as_bytes().iter().enumerate().all(|(index, byte)| matches!(index, 2 | 5) || byte.is_ascii_digit()) { return None; } let rest = &value[19..]; if rest.is_empty() || is_timezone_suffix(rest) { return Some(OracleExportDateParts { date, time, fraction: None, zone: (!rest.is_empty()).then_some(rest) }); } if let Some(after_dot) = rest.strip_prefix('.') { let digit_count = after_dot.bytes().take_while(|byte| byte.is_ascii_digit()).count(); if digit_count == 0 { return None; } let zone = &after_dot[digit_count..]; if zone.is_empty() || is_timezone_suffix(zone) { return Some(OracleExportDateParts { date, time, fraction: Some(&value[19..19 + 1 + digit_count]), zone: (!zone.is_empty()).then_some(zone), }); } } None } pub fn format_pg_array_sql_literal(arr: &[serde_json::Value]) -> String { if arr.is_empty() { return "'{}'".to_string(); } let elements: Vec = arr.iter().map(format_pg_array_element).collect(); let inner = format!("{{{}}}", elements.join(",")); format!("'{}'", inner.replace('\\', "\\\\").replace('\'', "''")) } pub(crate) fn is_postgres_vector_type(column_type: Option<&str>) -> bool { column_type .map(|column_type| { let normalized = column_type.trim().trim_matches('"').to_ascii_lowercase(); if normalized.trim_end().ends_with("[]") { return false; } let base = normalized.split(['(', ' ', '\t', '\n']).next().unwrap_or("").trim_matches('"'); matches!(base, "vector" | "halfvec") || base.ends_with(".vector") || base.ends_with(".halfvec") }) .unwrap_or(false) } pub(crate) fn format_postgres_vector_sql_literal(value: &serde_json::Value) -> String { if value.is_null() { return "NULL".to_string(); } let text = match value { // pgvector vector/halfvec are scalar extension types whose importable // literal grammar uses square brackets, unlike PostgreSQL arrays. serde_json::Value::Array(arr) => { let elements = arr.iter().map(format_postgres_vector_element).collect::>(); format!("[{}]", elements.join(",")) } serde_json::Value::String(text) => text.to_string(), _ => value.to_string(), }; quote_postgres_string_literal(&text) } fn format_postgres_vector_element(value: &serde_json::Value) -> String { match value { serde_json::Value::String(text) => text.trim().to_string(), serde_json::Value::Number(number) => number.to_string(), serde_json::Value::Bool(value) => value.to_string(), serde_json::Value::Null => "NULL".to_string(), _ => value.to_string(), } } fn format_pg_array_element(val: &serde_json::Value) -> String { match val { serde_json::Value::Null => "NULL".to_string(), serde_json::Value::Array(arr) => { if arr.is_empty() { return "{}".to_string(); } let elements: Vec = arr.iter().map(format_pg_array_element).collect(); format!("{{{}}}", elements.join(",")) } serde_json::Value::String(s) => { let escaped = s.replace('\\', "\\\\").replace('"', "\\\""); format!("\"{}\"", escaped) } serde_json::Value::Number(n) => n.to_string(), serde_json::Value::Bool(b) => { if *b { "true".to_string() } else { "false".to_string() } } serde_json::Value::Object(o) => { let json = serde_json::to_string(o).unwrap_or_default(); let escaped = json.replace('\\', "\\\\").replace('"', "\\\""); format!("\"{}\"", escaped) } } } pub fn format_ch_array_sql_literal(arr: &[serde_json::Value]) -> String { if arr.is_empty() { return "[]".to_string(); } let elements: Vec = arr.iter().map(format_ch_array_element).collect(); format!("[{}]", elements.join(",")) } fn format_ch_array_element(val: &serde_json::Value) -> String { match val { serde_json::Value::Null => "NULL".to_string(), serde_json::Value::Array(arr) => { if arr.is_empty() { return "[]".to_string(); } let elements: Vec = arr.iter().map(format_ch_array_element).collect(); format!("[{}]", elements.join(",")) } serde_json::Value::String(s) => { let escaped = s.replace('\\', "\\\\").replace('\'', "''"); format!("'{}'", escaped) } serde_json::Value::Number(n) => n.to_string(), serde_json::Value::Bool(b) => { if *b { "true".to_string() } else { "false".to_string() } } serde_json::Value::Object(o) => { let json = serde_json::to_string(o).unwrap_or_default(); format!("'{}'", json.replace('\\', "\\\\").replace('\'', "''")) } } } fn format_literal_string(value: &str, db_type: &DatabaseType, column_type: Option<&str>) -> String { if *db_type == DatabaseType::SqlServer { crate::sqlserver_temporal::normalize_sqlserver_temporal_literal(value, column_type) .unwrap_or_else(|| value.to_string()) } else if is_mysql_datetime_literal_database(db_type) && column_type.map(is_temporal_column_type).unwrap_or(true) { normalize_mysql_temporal_literal(value, column_type).unwrap_or_else(|| value.to_string()) } else { value.to_string() } } fn is_mysql_datetime_literal_database(db_type: &DatabaseType) -> bool { matches!( db_type, DatabaseType::Mysql | DatabaseType::Doris | DatabaseType::StarRocks | DatabaseType::Goldendb | DatabaseType::Sundb ) } fn normalize_mysql_temporal_literal(value: &str, column_type: Option<&str>) -> Option { let bytes = value.as_bytes(); if bytes.len() < 20 || !is_mysql_datetime_base(bytes) { return None; } let rest = &value[19..]; let (fraction, offset) = if let Some(after_dot) = rest.strip_prefix('.') { let digit_count = after_dot.bytes().take_while(|b| b.is_ascii_digit()).count(); if digit_count == 0 { return None; } let fraction_len = 1 + digit_count; (&rest[..fraction_len.min(7)], &rest[fraction_len..]) } else { ("", rest) }; if !is_timezone_suffix(offset) { return None; } match temporal_column_kind(column_type) { Some("date") => Some(value[..10].to_string()), Some("time") => Some(format!("{}{}", &value[11..19], fraction)), _ => Some(format!("{} {}{}", &value[..10], &value[11..19], fraction)), } } fn is_temporal_column_type(column_type: &str) -> bool { temporal_column_kind(Some(column_type)).is_some() } fn temporal_column_kind(column_type: Option<&str>) -> Option<&'static str> { let base = column_type?.trim().to_ascii_lowercase(); let base = base.split(['(', ':', ' ']).next().unwrap_or(""); match base { "date" => Some("date"), "time" => Some("time"), "datetime" | "timestamp" => Some("datetime"), _ => None, } } fn is_mysql_datetime_base(bytes: &[u8]) -> bool { matches!( bytes, [ y0, y1, y2, y3, b'-', m0, m1, b'-', d0, d1, sep, h0, h1, b':', min0, min1, b':', s0, s1, .. ] if y0.is_ascii_digit() && y1.is_ascii_digit() && y2.is_ascii_digit() && y3.is_ascii_digit() && m0.is_ascii_digit() && m1.is_ascii_digit() && d0.is_ascii_digit() && d1.is_ascii_digit() && (*sep == b'T' || *sep == b' ') && h0.is_ascii_digit() && h1.is_ascii_digit() && min0.is_ascii_digit() && min1.is_ascii_digit() && s0.is_ascii_digit() && s1.is_ascii_digit() ) } fn is_timezone_suffix(value: &str) -> bool { if value.eq_ignore_ascii_case("z") { return true; } let bytes = value.as_bytes(); matches!( bytes, [sign, h0, h1, b':', m0, m1] if (*sign == b'+' || *sign == b'-') && h0.is_ascii_digit() && h1.is_ascii_digit() && m0.is_ascii_digit() && m1.is_ascii_digit() ) } fn transfer_length_params(source_type: &str, target_db: &DatabaseType) -> String { let params = &source_type[source_type.find('(').expect("caller checked length parameters")..]; if matches!(target_db, DatabaseType::Oracle | DatabaseType::OceanbaseOracle | DatabaseType::Dameng) { params.to_string() } else { // Oracle length-unit qualifiers are invalid for non-Oracle-family // targets, which only accept the numeric length. normalize_len_params(params) } } pub fn map_column_type(source_type: &str, _source_db: &DatabaseType, target_db: &DatabaseType) -> String { if _source_db == target_db { return source_type.to_string(); } let t = source_type.to_lowercase(); let mut base = t.split('(').next().unwrap_or(&t).trim(); // Extract basic type, `bigint unsigned` -> `bigint` base = base.split(' ').next().unwrap_or(base).trim(); if matches!(target_db, DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala | DatabaseType::Argo) { return match base { "tinyint" => "TINYINT".into(), "smallint" | "int2" => "SMALLINT".into(), "int" | "integer" | "int4" | "mediumint" | "serial" | "smallserial" => "INT".into(), "bigint" | "int8" | "bigserial" => "BIGINT".into(), "float" | "float4" | "real" => "FLOAT".into(), "double" | "double precision" | "float8" => "DOUBLE".into(), "decimal" | "numeric" | "number" => { if let Some(index) = t.find('(') { format!("DECIMAL{}", &t[index..]) } else { "DECIMAL".into() } } "bool" | "boolean" | "bit" => "BOOLEAN".into(), "date" => "DATE".into(), "datetime" | "timestamp" | "timestamptz" | "timestamp with time zone" | "timestamp without time zone" => { "TIMESTAMP".into() } "binary" | "varbinary" | "blob" | "tinyblob" | "mediumblob" | "longblob" | "bytea" | "image" => { if matches!(target_db, DatabaseType::Impala) { "STRING".into() } else { "BINARY".into() } } _ => "STRING".into(), }; } match base { "int" | "integer" | "int4" | "mediumint" => match target_db { target_db if is_postgres_transfer_dialect(target_db) => "INTEGER".into(), DatabaseType::Mysql => "INT".into(), DatabaseType::SqlServer => "INT".into(), _ => "INTEGER".into(), }, "bigint" | "int8" => "BIGINT".into(), "smallint" | "int2" => "SMALLINT".into(), "tinyint" => match target_db { target_db if is_postgres_transfer_dialect(target_db) => "SMALLINT".into(), _ => "TINYINT".into(), }, "serial" | "bigserial" | "smallserial" => match target_db { target_db if is_postgres_transfer_dialect(target_db) => source_type.to_uppercase(), DatabaseType::Mysql => "BIGINT AUTO_INCREMENT".into(), _ => "INTEGER".into(), }, "float" | "float4" | "real" => match target_db { target_db if is_postgres_transfer_dialect(target_db) => "REAL".into(), _ => "FLOAT".into(), }, "double" | "double precision" | "float8" => match target_db { target_db if is_postgres_transfer_dialect(target_db) => "DOUBLE PRECISION".into(), _ => "DOUBLE".into(), }, "decimal" | "numeric" | "number" => { if t.contains('(') { match target_db { DatabaseType::Mysql | DatabaseType::SqlServer | DatabaseType::Oracle => { format!("DECIMAL{}", &t[t.find('(').unwrap()..]) } target_db if is_postgres_transfer_dialect(target_db) => { format!("DECIMAL{}", &t[t.find('(').unwrap()..]) } _ => "NUMERIC".into(), } } else { "NUMERIC".into() } } "varchar" | "nvarchar" | "character varying" | "varchar2" => { if t.contains('(') { let len_part = transfer_length_params(&t, target_db); match target_db { target_db if is_postgres_transfer_dialect(target_db) => format!("VARCHAR{len_part}"), DatabaseType::Mysql => format!("VARCHAR{len_part}"), DatabaseType::SqlServer => format!("NVARCHAR{len_part}"), _ => format!("VARCHAR{len_part}"), } } else { "VARCHAR(255)".into() } } "char" | "nchar" | "character" => { if t.contains('(') { let len_part = transfer_length_params(&t, target_db); format!("CHAR{len_part}") } else { "CHAR(1)".into() } } "longtext" => match target_db { DatabaseType::Mysql => "LONGTEXT".into(), _ => "TEXT".into(), }, "mediumtext" => match target_db { DatabaseType::Mysql => "MEDIUMTEXT".into(), _ => "TEXT".into(), }, "text" | "tinytext" | "clob" | "ntext" => "TEXT".into(), "bool" | "boolean" => match target_db { DatabaseType::Mysql => "TINYINT(1)".into(), DatabaseType::SqlServer => "BIT".into(), _ => "BOOLEAN".into(), }, "date" => "DATE".into(), "time" => "TIME".into(), "datetime" => match target_db { target_db if is_postgres_transfer_dialect(target_db) => "TIMESTAMP".into(), DatabaseType::ClickHouse => "DateTime64(6)".into(), _ => "DATETIME".into(), }, "timestamp" | "timestamptz" | "timestamp with time zone" | "timestamp without time zone" => match target_db { DatabaseType::Mysql => "DATETIME".into(), DatabaseType::SqlServer => "DATETIME2".into(), DatabaseType::ClickHouse => "DateTime64(6)".into(), _ => "TIMESTAMP".into(), }, "longblob" => match target_db { DatabaseType::Mysql => "LONGBLOB".into(), target_db if is_postgres_transfer_dialect(target_db) => "BYTEA".into(), DatabaseType::SqlServer => "VARBINARY(MAX)".into(), _ => "BLOB".into(), }, "mediumblob" => match target_db { DatabaseType::Mysql => "MEDIUMBLOB".into(), target_db if is_postgres_transfer_dialect(target_db) => "BYTEA".into(), DatabaseType::SqlServer => "VARBINARY(MAX)".into(), _ => "BLOB".into(), }, "blob" | "tinyblob" | "binary" | "varbinary" | "image" => match target_db { target_db if is_postgres_transfer_dialect(target_db) => "BYTEA".into(), DatabaseType::Mysql => "BLOB".into(), DatabaseType::SqlServer => "VARBINARY(MAX)".into(), _ => "BLOB".into(), }, "bytea" => match target_db { target_db if is_postgres_transfer_dialect(target_db) => "BYTEA".into(), DatabaseType::Mysql => "BLOB".into(), _ => "BLOB".into(), }, "json" | "jsonb" => match target_db { target_db if is_postgres_transfer_dialect(target_db) => "JSONB".into(), DatabaseType::Mysql => "JSON".into(), _ => "TEXT".into(), }, "uuid" => match target_db { target_db if is_postgres_transfer_dialect(target_db) => "UUID".into(), _ => "VARCHAR(36)".into(), }, "bit" => match target_db { target_db if is_postgres_transfer_dialect(target_db) => "BOOLEAN".into(), _ => "BIT".into(), }, _ => "TEXT".into(), } } fn mysql_type_needs_key_prefix(mapped_type: &str) -> bool { let base = mapped_type.split('(').next().unwrap_or(mapped_type).trim().to_ascii_lowercase(); matches!( base.as_str(), "text" | "tinytext" | "mediumtext" | "longtext" | "blob" | "tinyblob" | "mediumblob" | "longblob" ) } fn parse_mysql_row_error(error: &str) -> Option { let error = error.trim(); let at_row = error.rsplit("at row ").next()?; at_row.trim().parse::().ok() } pub fn generate_create_table_ddl( columns: &[db::ColumnInfo], table: &str, source_schema: &str, schema: &str, target_db: &DatabaseType, source_db: &DatabaseType, table_comment: Option<&str>, catalog: Option<&str>, ) -> String { generate_create_table_ddl_with_column_quoting( columns, table, source_schema, schema, target_db, source_db, table_comment, catalog, true, ) } #[allow(clippy::too_many_arguments)] fn generate_create_table_ddl_with_column_quoting( columns: &[db::ColumnInfo], table: &str, source_schema: &str, schema: &str, target_db: &DatabaseType, source_db: &DatabaseType, table_comment: Option<&str>, catalog: Option<&str>, quote_target_column_names: bool, ) -> String { let full_table = qualified_table(table, schema, target_db, catalog); let is_mysql_family = matches!( target_db, DatabaseType::Mysql | DatabaseType::Doris | DatabaseType::StarRocks | DatabaseType::Goldendb | DatabaseType::Sundb ); let mut col_lines = Vec::with_capacity(columns.len()); for c in columns { col_lines.push({ let mapped_type = postgres_column_type_sql(c, source_schema, schema, source_db, target_db); let mut line = format!( " {} {}", transfer_column_identifier(&c.name, target_db, quote_target_column_names), mapped_type ); if let Some(default_clause) = column_default_clause(c, source_schema, schema, source_db, target_db) { line.push(' '); line.push_str(&default_clause); } if !c.is_nullable && !matches!( target_db, DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala | DatabaseType::Argo ) { line.push_str(" NOT NULL"); } if is_mysql_family { let extra_clauses = parse_mysql_extra_clauses(c.extra.as_deref()); if extra_clauses.auto_increment { line.push_str(" AUTO_INCREMENT"); } if let Some(on_update_expr) = extra_clauses.on_update { line.push_str(&format!(" ON UPDATE {on_update_expr}")); } if let Some(ref comment) = c.comment { let trimmed = comment.trim(); if !trimmed.is_empty() { line.push_str(&format!(" COMMENT '{}'", trimmed.replace('\'', "''"))); } } } line }); } let mut pks = Vec::with_capacity(columns.iter().filter(|c| c.is_primary_key).count()); if !matches!(target_db, DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala | DatabaseType::Argo) { for c in columns { if c.is_primary_key { let qname = transfer_column_identifier(&c.name, target_db, quote_target_column_names); if is_mysql_family { let mapped = map_column_type(&c.data_type, source_db, target_db); if mysql_type_needs_key_prefix(&mapped) { pks.push(format!("{qname}(255)")); continue; } } pks.push(qname); } } } let mut ddl = match target_db { DatabaseType::SqlServer => { format!("IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '{table}')\n") } _ => String::new(), }; let create_prefix = match target_db { DatabaseType::SqlServer | DatabaseType::Dameng => "CREATE TABLE", _ => "CREATE TABLE IF NOT EXISTS", }; ddl.push_str(&format!("{create_prefix} {full_table} (\n")); ddl.push_str(&col_lines.join(",\n")); // ClickHouse: PRIMARY KEY must be a prefix of ORDER BY; skip inline PK // and encode it in the ENGINE clause below instead. if !pks.is_empty() && !matches!(target_db, DatabaseType::ClickHouse) { ddl.push_str(&format!(",\n PRIMARY KEY ({})", pks.join(", "))); } ddl.push_str("\n)"); if is_mysql_family { if let Some(comment) = table_comment { let trimmed = comment.trim(); if !trimmed.is_empty() { ddl.push_str(&format!(" COMMENT='{}'", trimmed.replace('\'', "''"))); } } } if matches!(target_db, DatabaseType::ClickHouse) { if pks.is_empty() { ddl.push_str(" ENGINE = MergeTree() ORDER BY tuple()"); } else { ddl.push_str(&format!(" ENGINE = MergeTree() ORDER BY ({})", pks.join(", "))); } } ddl } /// Generate COMMENT ON COLUMN / ALTER TABLE COMMENT COLUMN / COMMENT ON TABLE /// statements for databases that don't support inline comments in CREATE TABLE. /// MySQL family uses inline syntax (handled in generate_create_table_ddl). pub fn generate_comment_ddl( columns: &[db::ColumnInfo], table: &str, schema: &str, target_db: &DatabaseType, table_comment: Option<&str>, ) -> Vec { generate_comment_ddl_with_column_quoting(columns, table, schema, target_db, table_comment, true) } fn generate_comment_ddl_with_column_quoting( columns: &[db::ColumnInfo], table: &str, schema: &str, target_db: &DatabaseType, table_comment: Option<&str>, quote_target_column_names: bool, ) -> Vec { if !(is_postgres_transfer_dialect(target_db) || matches!(target_db, DatabaseType::Oracle | DatabaseType::ClickHouse)) { return Vec::new(); } let full_table = qualified_table(table, schema, target_db, None); let mut statements = Vec::new(); // Table-level comment first (PostgreSQL/Oracle only; ClickHouse doesn't support COMMENT ON TABLE) if is_postgres_transfer_dialect(target_db) || matches!(target_db, DatabaseType::Oracle) { if let Some(comment) = table_comment { let trimmed = comment.trim(); if !trimmed.is_empty() { let escaped = trimmed.replace('\'', "''"); statements.push(format!("COMMENT ON TABLE {full_table} IS '{escaped}'")); } } } for c in columns { if let Some(ref comment) = c.comment { let trimmed = comment.trim(); if trimmed.is_empty() { continue; } let escaped = trimmed.replace('\'', "''"); let qcol = transfer_column_identifier(&c.name, target_db, quote_target_column_names); match target_db { target_db if is_postgres_transfer_dialect(target_db) || matches!(target_db, DatabaseType::Oracle) => { statements.push(format!("COMMENT ON COLUMN {full_table}.{qcol} IS '{escaped}'")); } DatabaseType::ClickHouse => { statements.push(format!("ALTER TABLE {full_table} COMMENT COLUMN {qcol} '{escaped}'")); } _ => {} } } } statements } pub fn generate_insert( columns: &[String], rows: &[Vec], table: &str, schema: &str, db_type: &DatabaseType, ) -> String { generate_insert_typed(columns, &vec![None; columns.len()], rows, table, schema, db_type, None) } pub fn generate_insert_typed( columns: &[String], column_types: &[Option], rows: &[Vec], table: &str, schema: &str, db_type: &DatabaseType, catalog: Option<&str>, ) -> String { if rows.is_empty() { return String::new(); } let value_rows = value_rows_sql(rows, column_types, db_type, false); generate_insert_typed_from_value_rows(columns, &value_rows, table, schema, db_type, catalog) } pub(crate) fn generate_insert_typed_from_value_rows( columns: &[String], value_rows: &[String], table: &str, schema: &str, db_type: &DatabaseType, catalog: Option<&str>, ) -> String { InsertSqlTemplate::new(columns, table, schema, db_type, catalog, false).build(value_rows) } #[derive(Debug)] struct InsertSqlTemplate { standard_prefix: String, oracle_into_prefix: Option, } impl InsertSqlTemplate { fn new( columns: &[String], table: &str, schema: &str, db_type: &DatabaseType, catalog: Option<&str>, overrides_postgres_system_values: bool, ) -> Self { Self::new_with_column_quoting(columns, table, schema, db_type, catalog, overrides_postgres_system_values, true) } #[allow(clippy::too_many_arguments)] fn new_with_column_quoting( columns: &[String], table: &str, schema: &str, db_type: &DatabaseType, catalog: Option<&str>, overrides_postgres_system_values: bool, quote_target_column_names: bool, ) -> Self { let full_table = qualified_table(table, schema, db_type, catalog); let col_list = columns .iter() .map(|column| transfer_column_identifier(column, db_type, quote_target_column_names)) .collect::>() .join(", "); let overriding = if overrides_postgres_system_values && matches!(db_type, DatabaseType::Postgres) { " OVERRIDING SYSTEM VALUE" } else { "" }; Self { standard_prefix: format!("INSERT INTO {full_table} ({col_list}){overriding} VALUES\n"), oracle_into_prefix: matches!(db_type, DatabaseType::Oracle) .then(|| format!("INTO {full_table} ({col_list}) VALUES ")), } } fn build(&self, value_rows: &[String]) -> String { if value_rows.is_empty() { return String::new(); } if let Some(into_prefix) = self.oracle_into_prefix.as_deref().filter(|_| value_rows.len() > 1) { let capacity = "INSERT ALL\n".len() + into_prefix.len().saturating_mul(value_rows.len()) + value_rows.iter().map(String::len).sum::() + value_rows.len().saturating_sub(1) + "\nSELECT 1 FROM dual".len(); let mut sql = String::with_capacity(capacity); sql.push_str("INSERT ALL\n"); for (index, values) in value_rows.iter().enumerate() { if index > 0 { sql.push('\n'); } sql.push_str(into_prefix); sql.push_str(values); } sql.push_str("\nSELECT 1 FROM dual"); return sql; } let capacity = self.standard_prefix.len() + value_rows.iter().map(String::len).sum::() + ",\n".len().saturating_mul(value_rows.len().saturating_sub(1)); let mut sql = String::with_capacity(capacity); sql.push_str(&self.standard_prefix); for (index, values) in value_rows.iter().enumerate() { if index > 0 { sql.push_str(",\n"); } sql.push_str(values); } sql } fn statement_bytes(&self, value_rows_bytes: usize, row_count: usize, db_type: &DatabaseType) -> usize { if let Some(into_prefix) = self.oracle_into_prefix.as_deref().filter(|_| row_count > 1) { return sql_text_bytes("INSERT ALL\n", db_type) .saturating_add(sql_text_bytes(into_prefix, db_type).saturating_mul(row_count)) .saturating_add(value_rows_bytes) .saturating_add(sql_text_bytes("\n", db_type).saturating_mul(row_count - 1)) .saturating_add(sql_text_bytes("\nSELECT 1 FROM dual", db_type)); } sql_text_bytes(&self.standard_prefix, db_type) .saturating_add(value_rows_bytes) .saturating_add(sql_text_bytes(",\n", db_type).saturating_mul(row_count.saturating_sub(1))) } } fn sql_text_bytes(sql: &str, db_type: &DatabaseType) -> usize { if matches!(db_type, DatabaseType::SqlServer) { sql.encode_utf16().count().saturating_mul(2) } else { sql.len() } } fn value_rows_sql( rows: &[Vec], column_types: &[Option], db_type: &DatabaseType, mysql_spatial_markers: bool, ) -> Vec { let mut out = Vec::with_capacity(rows.len()); for row in rows { let mut values = String::with_capacity(row.len().saturating_mul(16).saturating_add(2)); values.push('('); for (index, v) in row.iter().enumerate() { if index > 0 { values.push_str(", "); } let column_type = column_types.get(index).and_then(|value| value.as_deref()); let value = if mysql_spatial_markers { crate::database_export::format_mysql_spatial_export_literal(v, Some(*db_type), column_type) } else { None } .unwrap_or_else(|| escape_value_typed(v, db_type, column_type)); values.push_str(&value); } values.push(')'); out.push(values); } out } pub fn generate_upsert( columns: &[String], rows: &[Vec], table: &str, schema: &str, db_type: &DatabaseType, pk_columns: &[String], ) -> String { generate_upsert_typed(columns, &vec![None; columns.len()], rows, table, schema, db_type, pk_columns, None) } pub fn generate_upsert_typed( columns: &[String], column_types: &[Option], rows: &[Vec], table: &str, schema: &str, db_type: &DatabaseType, pk_columns: &[String], catalog: Option<&str>, ) -> String { generate_upsert_typed_for_transfer( columns, column_types, rows, table, schema, db_type, pk_columns, catalog, false, false, true, ) } #[allow(clippy::too_many_arguments)] fn generate_upsert_typed_for_transfer( columns: &[String], column_types: &[Option], rows: &[Vec], table: &str, schema: &str, db_type: &DatabaseType, pk_columns: &[String], catalog: Option<&str>, overrides_postgres_system_values: bool, mysql_spatial_markers: bool, quote_target_column_names: bool, ) -> String { if rows.is_empty() || pk_columns.is_empty() { return String::new(); } let full_table = qualified_table(table, schema, db_type, catalog); let col_list = columns .iter() .map(|column| transfer_column_identifier(column, db_type, quote_target_column_names)) .collect::>() .join(", "); let value_rows = value_rows_sql(rows, column_types, db_type, mysql_spatial_markers); let mut non_pk_columns = Vec::with_capacity(columns.len().saturating_sub(pk_columns.len())); for c in columns { if !pk_columns.contains(c) { non_pk_columns.push(c); } } match db_type { db_type if is_postgres_transfer_dialect(db_type) || matches!(db_type, DatabaseType::Sqlite | DatabaseType::CloudflareD1 | DatabaseType::DuckDb) => { let pk_list = pk_columns .iter() .map(|column| transfer_column_identifier(column, db_type, quote_target_column_names)) .collect::>() .join(", "); let overriding = if overrides_postgres_system_values && matches!(db_type, DatabaseType::Postgres) { " OVERRIDING SYSTEM VALUE" } else { "" }; let mut sql = format!("INSERT INTO {full_table} ({col_list}){overriding} VALUES\n{}", value_rows.join(",\n")); if non_pk_columns.is_empty() { sql.push_str(&format!("\nON CONFLICT ({pk_list}) DO NOTHING")); } else { let update_set = non_pk_columns .iter() .map(|c| { let qc = transfer_column_identifier(c, db_type, quote_target_column_names); format!("{qc} = EXCLUDED.{qc}") }) .collect::>() .join(", "); sql.push_str(&format!("\nON CONFLICT ({pk_list}) DO UPDATE SET {update_set}")); } sql } DatabaseType::Mysql | DatabaseType::Doris | DatabaseType::StarRocks => { let mut sql = format!("INSERT INTO {full_table} ({col_list}) VALUES\n{}", value_rows.join(",\n")); if non_pk_columns.is_empty() { sql.push_str("\nON DUPLICATE KEY UPDATE "); let first_pk = transfer_column_identifier(&pk_columns[0], db_type, quote_target_column_names); sql.push_str(&format!("{first_pk} = {first_pk}")); } else { let update_set = non_pk_columns .iter() .map(|c| { let qc = transfer_column_identifier(c, db_type, quote_target_column_names); format!("{qc} = VALUES({qc})") }) .collect::>() .join(", "); sql.push_str(&format!("\nON DUPLICATE KEY UPDATE {update_set}")); } sql } DatabaseType::SqlServer => { let src_col_list = columns .iter() .map(|column| transfer_column_identifier(column, db_type, quote_target_column_names)) .collect::>() .join(", "); let on_clause = pk_columns .iter() .map(|c| { let qc = transfer_column_identifier(c, db_type, quote_target_column_names); format!("target.{qc} = src.{qc}") }) .collect::>() .join(" AND "); let mut sql = format!( "MERGE INTO {full_table} AS target USING (VALUES\n{}\n) AS src ({src_col_list}) ON {on_clause}", value_rows.join(",\n") ); if !non_pk_columns.is_empty() { let update_set = non_pk_columns .iter() .map(|c| { let qc = transfer_column_identifier(c, db_type, quote_target_column_names); format!("target.{qc} = src.{qc}") }) .collect::>() .join(", "); sql.push_str(&format!("\nWHEN MATCHED THEN UPDATE SET {update_set}")); } let insert_cols = columns .iter() .map(|column| transfer_column_identifier(column, db_type, quote_target_column_names)) .collect::>() .join(", "); let insert_vals = columns .iter() .map(|column| format!("src.{}", transfer_column_identifier(column, db_type, quote_target_column_names))) .collect::>() .join(", "); sql.push_str(&format!("\nWHEN NOT MATCHED THEN INSERT ({insert_cols}) VALUES ({insert_vals});")); sql } DatabaseType::Oracle => { let mut using_rows = Vec::with_capacity(rows.len()); for row in rows { let mut vals = Vec::with_capacity(row.len().min(columns.len())); for (index, (v, c)) in row.iter().zip(columns.iter()).enumerate() { vals.push(format!( "{} AS {}", escape_value_typed(v, db_type, column_types.get(index).and_then(|value| value.as_deref())), transfer_column_identifier(c, db_type, quote_target_column_names) )); } using_rows.push(format!("SELECT {} FROM dual", vals.join(", "))); } let on_clause = pk_columns .iter() .map(|c| { let qc = transfer_column_identifier(c, db_type, quote_target_column_names); format!("t.{qc} = s.{qc}") }) .collect::>() .join(" AND "); let mut sql = format!("MERGE INTO {full_table} t USING ({}) s ON ({on_clause})", using_rows.join(" UNION ALL ")); if !non_pk_columns.is_empty() { let update_set = non_pk_columns .iter() .map(|c| { let qc = transfer_column_identifier(c, db_type, quote_target_column_names); format!("t.{qc} = s.{qc}") }) .collect::>() .join(", "); sql.push_str(&format!("\nWHEN MATCHED THEN UPDATE SET {update_set}")); } let insert_cols = columns .iter() .map(|column| transfer_column_identifier(column, db_type, quote_target_column_names)) .collect::>() .join(", "); let insert_vals = columns .iter() .map(|column| format!("s.{}", transfer_column_identifier(column, db_type, quote_target_column_names))) .collect::>() .join(", "); sql.push_str(&format!("\nWHEN NOT MATCHED THEN INSERT ({insert_cols}) VALUES ({insert_vals})")); sql } _ => { let template = InsertSqlTemplate::new_with_column_quoting( columns, table, schema, db_type, catalog, false, quote_target_column_names, ); template.build(&value_rows_sql(rows, column_types, db_type, mysql_spatial_markers)) } } } fn max_transfer_write_rows(db_type: &DatabaseType, mode: &TransferMode) -> usize { match (db_type, mode) { (DatabaseType::SqlServer, TransferMode::Append | TransferMode::Overwrite) => MAX_SQLSERVER_INSERT_ROWS, (DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala | DatabaseType::Argo, _) => 500, (DatabaseType::Oracle, TransferMode::Append | TransferMode::Overwrite) => MAX_ORACLE_INSERT_ALL_ROWS, (DatabaseType::Oracle, TransferMode::Upsert) => MAX_ORACLE_MERGE_ROWS, _ => usize::MAX, } } fn contains_oceanbase_mysql_table_options(sql: &str) -> bool { let (sql_without_literals_or_comments, _) = protect_sql_literals(sql, true); OCEANBASE_MYSQL_TABLE_OPTION_RE.is_match(&sql_without_literals_or_comments) } fn mysql_ddl_collation_names(sql: &str) -> Vec { let mut names = Vec::new(); map_mysql_ddl_code_spans(sql, |code| { for captures in MYSQL_COLLATE_CLAUSE_RE.captures_iter(code) { let name = captures[1].to_string(); if !names.iter().any(|existing: &String| existing.eq_ignore_ascii_case(&name)) { names.push(name); } } String::new() }); names } fn remove_unsupported_mysql_collations(sql: &str, supported: &HashSet) -> String { let supported = supported.iter().map(|name| name.to_ascii_lowercase()).collect::>(); map_mysql_ddl_code_spans(sql, |code| { MYSQL_COLLATE_CLAUSE_RE .replace_all(code, |captures: ®ex::Captures| { if supported.contains(&captures[1].to_ascii_lowercase()) { captures[0].to_string() } else { String::new() } }) .to_string() }) } fn mysql_collations_for_transfer_ddl_recovery( sql: &str, error: &str, target_db_type: &DatabaseType, reused_source_ddl: bool, ) -> Option> { if !reused_source_ddl || !matches!(target_db_type, DatabaseType::Mysql) || !error.to_ascii_lowercase().contains("unknown collation") || !sql.trim_start().to_ascii_uppercase().starts_with("CREATE TABLE ") { return None; } let names = mysql_ddl_collation_names(sql); (!names.is_empty()).then_some(names) } fn can_reuse_source_table_ddl( source_db_type: &DatabaseType, target_db_type: &DatabaseType, source_driver_profile: Option<&str>, target_driver_profile: Option<&str>, preserves_target_table_name: bool, ) -> bool { if db::oceanbase_mysql::is_profile(source_db_type, source_driver_profile) && !db::oceanbase_mysql::is_profile(target_db_type, target_driver_profile) { return false; } preserves_target_table_name && !matches!(target_db_type, DatabaseType::ClickHouse) && (source_db_type == target_db_type || (is_mysql_family_target(source_db_type) && is_mysql_family_target(target_db_type)) || (is_postgres_family_target(source_db_type) && is_postgres_family_target(target_db_type))) } fn strip_dameng_storage_clauses(sql: &str) -> String { map_sql_code_spans(sql, false, |code| { let mut output = String::with_capacity(code.len()); let bytes = code.as_bytes(); let mut position = 0; while position < bytes.len() { let Some(relative_start) = code[position..].to_ascii_uppercase().find("STORAGE") else { output.push_str(&code[position..]); break; }; let start = position + relative_start; let before_is_identifier = start > 0 && (bytes[start - 1].is_ascii_alphanumeric() || matches!(bytes[start - 1], b'_' | b'$' | b'#')); let keyword_end = start + "STORAGE".len(); let after_is_identifier = keyword_end < bytes.len() && (bytes[keyword_end].is_ascii_alphanumeric() || matches!(bytes[keyword_end], b'_' | b'$' | b'#')); if before_is_identifier || after_is_identifier { output.push_str(&code[position..keyword_end]); position = keyword_end; continue; } let mut open = keyword_end; while open < bytes.len() && bytes[open].is_ascii_whitespace() { open += 1; } if open >= bytes.len() || bytes[open] != b'(' { output.push_str(&code[position..keyword_end]); position = keyword_end; continue; } let mut depth = 1usize; let mut end = open + 1; while end < bytes.len() && depth > 0 { match bytes[end] { b'(' => depth += 1, b')' => depth -= 1, _ => {} } end += 1; } if depth != 0 { output.push_str(&code[position..keyword_end]); position = keyword_end; continue; } let clause_start = position + code[position..start].trim_end().len(); output.push_str(&code[position..clause_start]); position = end; } output }) } fn rewrite_transfer_source_table_ddl( sql: &str, source_schema: &str, target_schema: &str, source_db_type: &DatabaseType, target_db_type: &DatabaseType, ) -> String { if is_postgres_family_target(source_db_type) && is_postgres_family_target(target_db_type) { rewrite_postgres_schema_qualified_references(sql, source_schema, target_schema) } else if matches!((source_db_type, target_db_type), (DatabaseType::Dameng, DatabaseType::Dameng)) { strip_dameng_storage_clauses(&rewrite_double_quoted_schema_qualifier(sql, source_schema, target_schema)) } else { sql.to_string() } } fn mysql_spatial_transfer_select_sql( sql: String, columns: &[String], column_types: &[Option], source_db_type: &DatabaseType, target_db_type: &DatabaseType, ) -> (String, bool) { let has_spatial_columns = column_types .iter() .any(|column_type| column_type.as_deref().is_some_and(crate::database_export::is_mysql_spatial_export_type)); if !matches!((source_db_type, target_db_type), (DatabaseType::Mysql, DatabaseType::Mysql)) || !has_spatial_columns { return (sql, false); } (crate::database_export::replace_database_export_select_list(sql, columns, column_types, source_db_type), true) } #[allow(clippy::too_many_arguments)] fn generate_transfer_write_sql( mode: &TransferMode, columns: &[String], column_types: &[Option], rows: &[Vec], table: &str, schema: &str, db_type: &DatabaseType, pk_columns: &[String], catalog: Option<&str>, overrides_postgres_system_values: bool, mysql_spatial_markers: bool, quote_target_column_names: bool, ) -> String { match mode { TransferMode::Upsert => generate_upsert_typed_for_transfer( columns, column_types, rows, table, schema, db_type, pk_columns, catalog, overrides_postgres_system_values, mysql_spatial_markers, quote_target_column_names, ), _ => { if rows.is_empty() { return String::new(); } let template = InsertSqlTemplate::new_with_column_quoting( columns, table, schema, db_type, catalog, overrides_postgres_system_values, quote_target_column_names, ); template.build(&value_rows_sql(rows, column_types, db_type, mysql_spatial_markers)) } } } #[allow(clippy::too_many_arguments)] #[cfg_attr(not(test), allow(dead_code))] pub(crate) fn generate_insert_typed_sql_batches( columns: &[String], column_types: &[Option], rows: &[Vec], table: &str, schema: &str, db_type: &DatabaseType, catalog: Option<&str>, limits: SqlBatchLimits, ) -> Result, String> { let value_rows = value_rows_sql(rows, column_types, db_type, false); generate_insert_typed_sql_batches_from_value_rows(columns, &value_rows, table, schema, db_type, catalog, limits) } #[allow(clippy::too_many_arguments)] pub(crate) fn generate_insert_typed_sql_batches_from_value_rows( columns: &[String], value_rows: &[String], table: &str, schema: &str, db_type: &DatabaseType, catalog: Option<&str>, limits: SqlBatchLimits, ) -> Result, String> { generate_insert_sql_batches_from_value_rows( columns, value_rows, table, schema, db_type, catalog, limits, false, true, ) } #[allow(clippy::too_many_arguments)] fn generate_insert_typed_sql_batches_for_transfer( columns: &[String], column_types: &[Option], rows: &[Vec], table: &str, schema: &str, db_type: &DatabaseType, catalog: Option<&str>, limits: SqlBatchLimits, overrides_postgres_system_values: bool, mysql_spatial_markers: bool, quote_target_column_names: bool, ) -> Result, String> { if rows.is_empty() { return Ok(Vec::new()); } let value_rows = value_rows_sql(rows, column_types, db_type, mysql_spatial_markers); generate_insert_sql_batches_from_value_rows( columns, &value_rows, table, schema, db_type, catalog, limits, overrides_postgres_system_values, quote_target_column_names, ) } #[allow(clippy::too_many_arguments)] fn generate_insert_sql_batches_from_value_rows( columns: &[String], value_rows: &[String], table: &str, schema: &str, db_type: &DatabaseType, catalog: Option<&str>, limits: SqlBatchLimits, overrides_postgres_system_values: bool, quote_target_column_names: bool, ) -> Result, String> { if value_rows.is_empty() { return Ok(Vec::new()); } let max_rows = limits.max_rows.max(1).min(match db_type { DatabaseType::SqlServer => MAX_SQLSERVER_INSERT_ROWS, DatabaseType::Oracle => MAX_ORACLE_INSERT_ALL_ROWS, _ => usize::MAX, }); let target_sql_bytes = limits.target_sql_bytes.max(1); let batch_sql_bytes = limits.hard_sql_bytes.map_or(target_sql_bytes, |hard| target_sql_bytes.min(hard)); let template = InsertSqlTemplate::new_with_column_quoting( columns, table, schema, db_type, catalog, overrides_postgres_system_values, quote_target_column_names, ); let value_row_bytes = value_rows.iter().map(|row| sql_text_bytes(row, db_type)).collect::>(); let mut statements = Vec::new(); let mut start = 0usize; while start < value_rows.len() { let mut end = start; let mut rows_bytes = 0usize; while end < value_rows.len() && end - start < max_rows { let single_row_bytes = template.statement_bytes(value_row_bytes[end], 1, db_type); if let Some(hard_sql_bytes) = limits.hard_sql_bytes { if single_row_bytes > hard_sql_bytes { return Err(format!( "SQL batch row {} requires {} bytes and exceeds the {} byte hard limit", end + 1, single_row_bytes, hard_sql_bytes )); } } let candidate_rows_bytes = rows_bytes.saturating_add(value_row_bytes[end]); let candidate_row_count = end - start + 1; let candidate_bytes = template.statement_bytes(candidate_rows_bytes, candidate_row_count, db_type); if candidate_row_count > 1 && candidate_bytes > batch_sql_bytes { break; } rows_bytes = candidate_rows_bytes; end += 1; } statements.push((template.build(&value_rows[start..end]), end - start)); start = end; } Ok(statements) } #[allow(clippy::too_many_arguments)] #[cfg_attr(not(test), allow(dead_code))] fn generate_transfer_write_sql_batches( mode: &TransferMode, columns: &[String], column_types: &[Option], rows: &[Vec], table: &str, schema: &str, db_type: &DatabaseType, pk_columns: &[String], catalog: Option<&str>, overrides_postgres_system_values: bool, mysql_spatial_markers: bool, ) -> Result, String> { generate_transfer_write_sql_batches_with_column_quoting( mode, columns, column_types, rows, table, schema, db_type, pk_columns, catalog, overrides_postgres_system_values, mysql_spatial_markers, true, ) } #[allow(clippy::too_many_arguments)] fn generate_transfer_write_sql_batches_with_column_quoting( mode: &TransferMode, columns: &[String], column_types: &[Option], rows: &[Vec], table: &str, schema: &str, db_type: &DatabaseType, pk_columns: &[String], catalog: Option<&str>, overrides_postgres_system_values: bool, mysql_spatial_markers: bool, quote_target_column_names: bool, ) -> Result, String> { if rows.is_empty() { return Ok(Vec::new()); } if matches!(mode, TransferMode::Append | TransferMode::Overwrite) { return Ok(generate_insert_typed_sql_batches_for_transfer( columns, column_types, rows, table, schema, db_type, catalog, SqlBatchLimits::for_database(db_type, max_transfer_write_rows(db_type, mode)), overrides_postgres_system_values, mysql_spatial_markers, quote_target_column_names, )? .into_iter() .map(|(sql, _)| sql) .collect()); } let max_rows = max_transfer_write_rows(db_type, mode); let max_sql_bytes = match db_type { DatabaseType::CloudflareD1 => crate::db::cloudflare_d1::MAX_SQL_STATEMENT_BYTES, _ => MAX_TRANSFER_WRITE_SQL_BYTES, }; let mut statements = Vec::new(); let mut start = 0; while start < rows.len() { let mut end = start + 1; let mut accepted = generate_transfer_write_sql( mode, columns, column_types, &rows[start..end], table, schema, db_type, pk_columns, catalog, overrides_postgres_system_values, mysql_spatial_markers, quote_target_column_names, ); while end < rows.len() && end - start < max_rows { let candidate = generate_transfer_write_sql( mode, columns, column_types, &rows[start..=end], table, schema, db_type, pk_columns, catalog, overrides_postgres_system_values, mysql_spatial_markers, quote_target_column_names, ); if candidate.len() > max_sql_bytes && !accepted.is_empty() { break; } accepted = candidate; end += 1; } if !accepted.is_empty() { statements.push(accepted); } start = end; } Ok(statements) } pub fn pagination_sql( columns: &[String], table: &str, schema: &str, db_type: &DatabaseType, offset: u64, limit: usize, ) -> String { let full_table = qualified_table(table, schema, db_type, None); let col_list = columns.iter().map(|c| quote_identifier(c, db_type)).collect::>().join(", "); match db_type { DatabaseType::Oracle | DatabaseType::OceanbaseOracle => { let base_sql = format!("SELECT {col_list} FROM {full_table}"); oracle_rownum_page_sql(&col_list, base_sql, offset, limit) } DatabaseType::Informix => { if offset == 0 { format!("SELECT FIRST {limit} {col_list} FROM {full_table}") } else { format!("SELECT SKIP {offset} FIRST {limit} {col_list} FROM {full_table}") } } DatabaseType::SqlServer => { sqlserver_row_number_page_sql(&col_list, &full_table, "(SELECT NULL)", offset, limit) } DatabaseType::Dameng => { format!( "SELECT {col_list} FROM {full_table} ORDER BY (SELECT NULL) OFFSET {offset} ROWS FETCH NEXT {limit} ROWS ONLY" ) } DatabaseType::Questdb => { let upper_bound = offset + limit as u64; format!("SELECT {col_list} FROM {full_table} LIMIT {offset}, {upper_bound}") } _ => { format!("SELECT {col_list} FROM {full_table} LIMIT {limit} OFFSET {offset}") } } } pub fn pagination_sql_with_order( columns: &[String], table: &str, schema: &str, db_type: &DatabaseType, offset: u64, limit: usize, order_by_columns: &[String], catalog: Option<&str>, ) -> String { let full_table = qualified_table(table, schema, db_type, catalog); let col_list = columns.iter().map(|c| quote_identifier(c, db_type)).collect::>().join(", "); let order_expression = postgres_order_by_expression(order_by_columns, db_type); match db_type { DatabaseType::Oracle | DatabaseType::OceanbaseOracle => { let order_by = order_expression.map(|value| format!(" ORDER BY {value}")).unwrap_or_default(); let base_sql = format!("SELECT {col_list} FROM {full_table}{order_by}"); oracle_rownum_page_sql(&col_list, base_sql, offset, limit) } DatabaseType::Informix => { let order_by = order_expression.map(|value| format!(" ORDER BY {value}")).unwrap_or_default(); if offset == 0 { format!("SELECT FIRST {limit} {col_list} FROM {full_table}{order_by}") } else { format!("SELECT SKIP {offset} FIRST {limit} {col_list} FROM {full_table}{order_by}") } } DatabaseType::SqlServer => { let order_by = order_expression.unwrap_or_else(|| "(SELECT NULL)".to_string()); sqlserver_row_number_page_sql(&col_list, &full_table, &order_by, offset, limit) } DatabaseType::Dameng => { let order_by = order_expression.unwrap_or_else(|| "(SELECT NULL)".to_string()); format!( "SELECT {col_list} FROM {full_table} ORDER BY {order_by} OFFSET {offset} ROWS FETCH NEXT {limit} ROWS ONLY" ) } DatabaseType::Questdb => { let upper_bound = offset + limit as u64; let order_by = order_expression.map(|value| format!(" ORDER BY {value}")).unwrap_or_default(); format!("SELECT {col_list} FROM {full_table}{order_by} LIMIT {offset}, {upper_bound}") } _ => { let order_by = order_expression.map(|value| format!(" ORDER BY {value}")).unwrap_or_default(); format!("SELECT {col_list} FROM {full_table}{order_by} LIMIT {limit} OFFSET {offset}") } } } #[allow(clippy::too_many_arguments)] pub fn pagination_sql_with_filter_order( columns: &[String], table: &str, schema: &str, db_type: &DatabaseType, offset: u64, limit: usize, where_input: Option<&str>, order_by: Option<&str>, default_order_columns: &[String], ) -> String { pagination_sql_with_filter_order_and_identifier_quote( columns, table, schema, db_type, offset, limit, where_input, order_by, default_order_columns, None, ) } #[allow(clippy::too_many_arguments)] pub fn pagination_sql_with_filter_order_and_identifier_quote( columns: &[String], table: &str, schema: &str, db_type: &DatabaseType, offset: u64, limit: usize, where_input: Option<&str>, order_by: Option<&str>, default_order_columns: &[String], identifier_quote: Option<&str>, ) -> String { let full_table = qualified_table_with_identifier_quote(table, schema, db_type, None, identifier_quote); let col_list = columns .iter() .map(|c| quote_identifier_with_identifier_quote(c, db_type, identifier_quote)) .collect::>() .join(", "); let predicate = crate::sql_dialect::normalize_where_input(where_input); let where_clause = if predicate.is_empty() { String::new() } else { format!(" WHERE ({predicate})") }; let order_expression = order_by.map(str::trim).filter(|value| !value.is_empty()).map(str::to_string).or_else(|| { postgres_order_by_expression_with_identifier_quote(default_order_columns, db_type, identifier_quote) }); match db_type { DatabaseType::Oracle | DatabaseType::OceanbaseOracle => { let order_by = order_expression.map(|value| format!(" ORDER BY {value}")).unwrap_or_default(); let base_sql = format!("SELECT {col_list} FROM {full_table}{where_clause}{order_by}"); oracle_rownum_page_sql(&col_list, base_sql, offset, limit) } DatabaseType::Informix => { let order_by = order_expression.map(|value| format!(" ORDER BY {value}")).unwrap_or_default(); if offset == 0 { format!("SELECT FIRST {limit} {col_list} FROM {full_table}{where_clause}{order_by}") } else { format!("SELECT SKIP {offset} FIRST {limit} {col_list} FROM {full_table}{where_clause}{order_by}") } } DatabaseType::SqlServer => { let order_by = order_expression.unwrap_or_else(|| "(SELECT NULL)".to_string()); let from_clause = format!("{full_table}{where_clause}"); sqlserver_row_number_page_sql(&col_list, &from_clause, &order_by, offset, limit) } DatabaseType::Dameng => { let order_by = order_expression.unwrap_or_else(|| "(SELECT NULL)".to_string()); format!( "SELECT {col_list} FROM {full_table}{where_clause} ORDER BY {order_by} OFFSET {offset} ROWS FETCH NEXT {limit} ROWS ONLY" ) } DatabaseType::Questdb => { let upper_bound = offset + limit as u64; let order_by = order_expression.map(|value| format!(" ORDER BY {value}")).unwrap_or_default(); format!("SELECT {col_list} FROM {full_table}{where_clause}{order_by} LIMIT {offset}, {upper_bound}") } _ => { let order_by = order_expression.map(|value| format!(" ORDER BY {value}")).unwrap_or_default(); format!("SELECT {col_list} FROM {full_table}{where_clause}{order_by} LIMIT {limit} OFFSET {offset}") } } } pub fn count_sql(table: &str, schema: &str, db_type: &DatabaseType, catalog: Option<&str>) -> String { count_sql_with_where(table, schema, db_type, None, catalog) } pub fn count_sql_with_where( table: &str, schema: &str, db_type: &DatabaseType, where_input: Option<&str>, catalog: Option<&str>, ) -> String { count_sql_with_where_and_identifier_quote(table, schema, db_type, where_input, catalog, None) } pub fn count_sql_with_where_and_identifier_quote( table: &str, schema: &str, db_type: &DatabaseType, where_input: Option<&str>, catalog: Option<&str>, identifier_quote: Option<&str>, ) -> String { let full_table = qualified_table_with_identifier_quote(table, schema, db_type, catalog, identifier_quote); let predicate = crate::sql_dialect::normalize_where_input(where_input); let where_clause = if predicate.is_empty() { String::new() } else { format!(" WHERE ({predicate})") }; format!("SELECT COUNT(*) FROM {full_table}{where_clause}") } pub fn keyset_pagination_sql( columns: &[String], table: &str, schema: &str, db_type: &DatabaseType, primary_keys: &[String], last_pk_values: &[serde_json::Value], limit: usize, ) -> String { keyset_pagination_sql_with_identifier_quote( columns, table, schema, db_type, primary_keys, last_pk_values, limit, None, ) } #[allow(clippy::too_many_arguments)] pub fn keyset_pagination_sql_with_identifier_quote( columns: &[String], table: &str, schema: &str, db_type: &DatabaseType, primary_keys: &[String], last_pk_values: &[serde_json::Value], limit: usize, identifier_quote: Option<&str>, ) -> String { let full_table = qualified_table_with_identifier_quote(table, schema, db_type, None, identifier_quote); let col_list = columns .iter() .map(|c| quote_identifier_with_identifier_quote(c, db_type, identifier_quote)) .collect::>() .join(", "); let order = primary_keys .iter() .map(|pk| format!("{} ASC", quote_identifier_with_identifier_quote(pk, db_type, identifier_quote))) .collect::>() .join(", "); let where_clause = keyset_where_clause(primary_keys, last_pk_values, db_type, identifier_quote); match db_type { DatabaseType::Oracle | DatabaseType::OceanbaseOracle => { let base_sql = format!("SELECT {col_list} FROM {full_table}{where_clause} ORDER BY {order}"); oracle_rownum_page_sql(&col_list, base_sql, 0, limit) } DatabaseType::Informix => { format!("SELECT FIRST {limit} {col_list} FROM {full_table}{where_clause} ORDER BY {order}") } DatabaseType::SqlServer => { format!("SELECT TOP ({limit}) {col_list} FROM {full_table}{where_clause} ORDER BY {order}") } DatabaseType::Dameng => { format!( "SELECT {col_list} FROM {full_table}{where_clause} ORDER BY {order} OFFSET 0 ROWS FETCH NEXT {limit} ROWS ONLY" ) } _ => { format!("SELECT {col_list} FROM {full_table}{where_clause} ORDER BY {order} LIMIT {limit}") } } } fn keyset_where_clause( primary_keys: &[String], last_pk_values: &[serde_json::Value], db_type: &DatabaseType, identifier_quote: Option<&str>, ) -> String { if primary_keys.is_empty() || last_pk_values.is_empty() { return String::new(); } let quoted_keys = primary_keys .iter() .map(|pk| quote_identifier_with_identifier_quote(pk, db_type, identifier_quote)) .collect::>(); let literals = last_pk_values.iter().map(|v| value_to_sql_literal(v, db_type)).collect::>(); let comparison_count = quoted_keys.len().min(literals.len()); if comparison_count == 0 { return String::new(); } let mut clauses = Vec::with_capacity(comparison_count); for index in 0..comparison_count { let mut parts = Vec::with_capacity(index + 1); for prefix_index in 0..index { parts.push(format!("{} = {}", quoted_keys[prefix_index], literals[prefix_index])); } parts.push(format!("{} > {}", quoted_keys[index], literals[index])); if parts.len() == 1 { clauses.push(parts.remove(0)); } else { clauses.push(format!("({})", parts.join(" AND "))); } } if clauses.len() == 1 { format!(" WHERE {}", clauses[0]) } else { format!(" WHERE ({})", clauses.join(" OR ")) } } fn value_to_sql_literal(value: &serde_json::Value, _db_type: &DatabaseType) -> String { match value { serde_json::Value::Null => "NULL".to_string(), serde_json::Value::Bool(b) => { if *b { "TRUE".to_string() } else { "FALSE".to_string() } } serde_json::Value::Number(n) => n.to_string(), serde_json::Value::String(s) => quote_string_literal(s), _ => quote_string_literal(&value.to_string()), } } fn is_mongodb_transfer_type(db_type: &DatabaseType) -> bool { matches!(db_type, DatabaseType::MongoDb) } fn mongo_transfer_document_fields(documents: &[serde_json::Value]) -> Vec { let mut fields = Vec::new(); let mut seen = HashSet::new(); for document in documents { let Some(object) = document.as_object() else { continue; }; for key in object.keys() { if seen.insert(key.clone()) { fields.push(key.clone()); } } } fields } fn mongo_documents_to_rows(documents: &[serde_json::Value], columns: &[String]) -> Vec> { documents .iter() .map(|document| { let object = document.as_object(); columns .iter() .map(|column| object.and_then(|values| values.get(column)).cloned().unwrap_or(serde_json::Value::Null)) .collect() }) .collect() } fn sql_rows_to_mongo_documents(columns: &[String], rows: &[Vec]) -> Vec { rows.iter() .map(|row| { let mut document = serde_json::Map::new(); for (index, column) in columns.iter().enumerate() { document.insert(column.clone(), row.get(index).cloned().unwrap_or(serde_json::Value::Null)); } serde_json::Value::Object(document) }) .collect() } /// Select the representation used by a MongoDB transfer. MongoDB-to-MongoDB /// writes consume canonical Extended JSON so BSON values (for example dates) /// keep their server-side types; SQL targets continue using the browser view. fn mongo_documents_for_transfer(result: MongoDocumentResult, preserve_bson_types: bool) -> Vec { let MongoDocumentResult { documents, extended_documents, .. } = result; if preserve_bson_types { match extended_documents { Some(extended_documents) if extended_documents.len() == documents.len() => extended_documents, _ => documents, } } else { documents } } async fn find_mongo_documents_extended_json( state: &AppState, connection_id: &str, database: &str, collection: &str, offset: u64, batch_size: usize, ) -> Result { crate::mongo_ops::mongo_find_documents_extended_json_core( state, connection_id, database, collection, offset, batch_size as i64, None, None, Some(r#"{"_id":1}"#), ) .await } async fn find_mongo_documents_for_rows( state: &AppState, connection_id: &str, database: &str, collection: &str, offset: u64, batch_size: usize, ) -> Result { crate::mongo_ops::mongo_find_documents_core( state, connection_id, database, collection, offset, batch_size as i64, None, None, Some(r#"{"_id":1}"#), None, ) .await } async fn insert_mongo_documents_for_transfer( state: &AppState, connection_id: &str, database: &str, collection: &str, documents: &[serde_json::Value], ) -> Result { if documents.is_empty() { return Ok(0); } let docs_json = serde_json::to_string(documents).map_err(|e| format!("Failed to encode MongoDB documents: {e}"))?; match crate::mongo_ops::mongo_insert_documents_core(state, connection_id, database, collection, &docs_json).await { Ok(count) => Ok(count), Err(error) if error.to_ascii_lowercase().contains("legacy agent") => { let mut inserted = 0; for document in documents { let doc_json = serde_json::to_string(document).map_err(|e| format!("Failed to encode MongoDB document: {e}"))?; crate::mongo_ops::mongo_insert_document_core(state, connection_id, database, collection, &doc_json) .await?; inserted += 1; } Ok(inserted) } Err(error) => Err(error), } } async fn insert_mongo_documents_extended_json_for_transfer( state: &AppState, connection_id: &str, database: &str, collection: &str, documents: &[serde_json::Value], ) -> Result { if documents.is_empty() { return Ok(0); } let docs_json = serde_json::to_string(documents).map_err(|e| format!("Failed to encode MongoDB documents: {e}"))?; match crate::mongo_ops::mongo_insert_documents_extended_json_core( state, connection_id, database, collection, &docs_json, ) .await { Ok(count) => Ok(count), Err(error) if error.to_ascii_lowercase().contains("legacy agent") => { insert_mongo_documents_for_transfer(state, connection_id, database, collection, documents).await } Err(error) => Err(error), } } async fn overwrite_mongo_collection_for_transfer( state: &AppState, connection_id: &str, database: &str, collection: &str, ) -> Result<(), String> { crate::mongo_ops::mongo_delete_documents_core(state, connection_id, database, collection, "{}", true) .await .map(|_| ()) } fn mongo_value_column_type(value: Option<&serde_json::Value>) -> String { match value { Some(serde_json::Value::Bool(_)) => "boolean".to_string(), Some(serde_json::Value::Number(number)) if number.is_i64() || number.is_u64() => "bigint".to_string(), Some(serde_json::Value::Number(_)) => "double".to_string(), Some(serde_json::Value::Array(_) | serde_json::Value::Object(_)) => "json".to_string(), _ => "text".to_string(), } } fn mongo_columns_from_documents(documents: &[serde_json::Value]) -> Vec { mongo_transfer_document_fields(documents) .into_iter() .map(|name| { let sample = documents.iter().filter_map(|document| document.as_object()?.get(&name)).find(|value| !value.is_null()); db::ColumnInfo { name, data_type: mongo_value_column_type(sample), is_nullable: true, column_default: None, is_primary_key: false, extra: None, comment: None, numeric_precision: None, numeric_scale: None, character_maximum_length: None, enum_values: None, ..Default::default() } }) .collect() } pub async fn execute_on_pool(state: &AppState, pool_key: &str, sql: &str) -> Result { execute_on_pool_with_options(state, pool_key, sql, None, TransferExecutionSafety::WriteNoReplay).await } pub async fn execute_read_on_pool(state: &AppState, pool_key: &str, sql: &str) -> Result { execute_read_on_pool_with_max_rows(state, pool_key, sql, None).await } pub async fn execute_read_on_pool_with_max_rows( state: &AppState, pool_key: &str, sql: &str, max_rows: Option, ) -> Result { execute_on_pool_with_options(state, pool_key, sql, max_rows, TransferExecutionSafety::ReadOnlyRetryable).await } async fn execute_transfer_ddl_on_pool( state: &AppState, pool_key: &str, sql: &str, db_type: &DatabaseType, ) -> Result<(), String> { for statement in transfer_ddl_statements(sql, db_type) { execute_on_pool(state, pool_key, &statement).await?; } Ok(()) } async fn supported_mysql_transfer_collations( state: &AppState, pool_key: &str, names: &[String], ) -> Result, String> { let names = names.iter().map(|name| quote_string_literal(name)).collect::>().join(", "); let sql = format!("SELECT COLLATION_NAME FROM information_schema.COLLATIONS WHERE COLLATION_NAME IN ({names})"); let result = execute_on_pool(state, pool_key, &sql).await?; Ok(result.rows.iter().filter_map(|row| json_string_cell(row, 0)).map(|name| name.to_ascii_lowercase()).collect()) } async fn execute_transfer_create_table_ddl_on_pool( state: &AppState, pool_key: &str, sql: &str, db_type: &DatabaseType, reused_source_ddl: bool, ) -> Result<(), String> { let original_error = match execute_transfer_ddl_on_pool(state, pool_key, sql, db_type).await { Ok(()) => return Ok(()), Err(error) => error, }; let Some(collations) = mysql_collations_for_transfer_ddl_recovery(sql, &original_error, db_type, reused_source_ddl) else { return Err(original_error); }; let supported = supported_mysql_transfer_collations(state, pool_key, &collations) .await .map_err(|error| format!("{original_error}; failed to inspect target MySQL collations: {error}"))?; let rewritten = remove_unsupported_mysql_collations(sql, &supported); if rewritten == sql { return Err(format!("{original_error}; target MySQL reports all referenced collations as supported")); } let unsupported = collations.iter().filter(|name| !supported.contains(&name.to_ascii_lowercase())).cloned().collect::>(); log::warn!("[transfer] retrying target table DDL without unsupported MySQL collations: {}", unsupported.join(", ")); execute_transfer_ddl_on_pool(state, pool_key, &rewritten, db_type) .await .map_err(|error| format!("{original_error}; retry without unsupported MySQL collations failed: {error}")) } fn transfer_table_already_exists_error(error: &str) -> bool { let lower = error.to_ascii_lowercase(); lower.contains("already exists") || lower.contains("there is already") || lower.contains("duplicate_table") || lower.contains("42p07") || error.contains("已经存在") || error.contains("已存在") } fn transfer_create_table_created(result: Result<(), String>, error_prefix: &str) -> Result { match result { Ok(_) => Ok(true), Err(e) if transfer_table_already_exists_error(&e) => Ok(false), Err(e) => Err(format!("{error_prefix}: {e}")), } } fn transfer_ddl_statements(sql: &str, db_type: &DatabaseType) -> Vec { if is_postgres_transfer_dialect(db_type) { let statements = split_sql_statements(sql); if statements.is_empty() { vec![sql.trim().to_string()] } else { statements .into_iter() .map(|statement| strip_inline_foreign_key_constraint_lines(&statement)) .filter(|statement| !is_postgres_post_table_index_statement(statement)) .collect() } } else if matches!(db_type, DatabaseType::Dameng) { let statements = split_sql_statements_for_database(sql, *db_type); if statements.is_empty() { vec![sql.trim().to_string()] } else { statements } } else { vec![sql.to_string()] } } /// Strips inline `CONSTRAINT ... FOREIGN KEY ... REFERENCES ...` lines from a /// `CREATE TABLE` statement, fixing up a trailing comma only when removing the /// foreign key leaves one immediately before the table's closing parenthesis. /// Dialect-agnostic: relies only on the definition-line shape shared by Postgres /// and MySQL-family DDL dumps. /// /// Only genuine constraint definition lines match (`CONSTRAINT FOREIGN /// KEY (` / bare `FOREIGN KEY (`); a column line whose COMMENT or DEFAULT text /// merely mentions the words must survive (#7660). fn strip_inline_foreign_key_constraint_lines(statement: &str) -> String { if !statement.trim_start().to_ascii_uppercase().starts_with("CREATE TABLE ") { return statement.to_string(); } let mut lines: Vec = Vec::new(); let mut removed_foreign_key = false; for line in statement.lines() { if INLINE_FOREIGN_KEY_CONSTRAINT_LINE_RE.is_match(line) { removed_foreign_key = true; continue; } lines.push(line.to_string()); } if removed_foreign_key { if let Some(closing_index) = lines.iter().rposition(|line| line.trim_start().starts_with(')')) { if let Some(previous) = lines[..closing_index].iter_mut().rfind(|line| !line.trim().is_empty()) { let trimmed_len = previous.trim_end_matches(char::is_whitespace).len(); if previous[..trimmed_len].ends_with(',') { previous.remove(trimmed_len - 1); } } } } lines.join("\n") } fn is_postgres_post_table_index_statement(statement: &str) -> bool { let normalized = statement.trim_start().to_ascii_uppercase(); normalized.starts_with("CREATE INDEX ") || normalized.starts_with("CREATE UNIQUE INDEX ") || normalized.starts_with("COMMENT ON INDEX ") } pub async fn execute_on_pool_with_max_rows( state: &AppState, pool_key: &str, sql: &str, max_rows: Option, ) -> Result { execute_on_pool_with_options(state, pool_key, sql, max_rows, TransferExecutionSafety::WriteNoReplay).await } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum TransferExecutionSafety { ReadOnlyRetryable, WriteNoReplay, } fn transfer_pool_error_action( safety: TransferExecutionSafety, db_type: Option, err: &str, ) -> PoolErrorAction { match (safety, pool_error_action(db_type, err)) { (TransferExecutionSafety::WriteNoReplay, PoolErrorAction::ReconnectAndRetry) => PoolErrorAction::Discard, (_, action) => action, } } async fn transfer_pool_context( state: &AppState, pool_key: &str, ) -> (Option, Option, Option) { let configs = state.configs.read().await; let config = config_for_pool_key(pool_key, &configs); ( config.map(|config| config.id.clone()), database_from_pool_key(pool_key).map(str::to_string), config.map(|config| config.db_type), ) } fn client_session_id_from_pool_key(pool_key: &str) -> Option<&str> { pool_key.split_once(":session:").map(|(_, session)| session).filter(|session| !session.is_empty()) } async fn execute_on_pool_with_options( state: &AppState, pool_key: &str, sql: &str, max_rows: Option, safety: TransferExecutionSafety, ) -> Result { let (connection_id, database, db_type) = transfer_pool_context(state, pool_key).await; let client_session_id = client_session_id_from_pool_key(pool_key).map(str::to_string); let mut current_pool_key = pool_key.to_string(); for attempt in 0..2 { let result = execute_on_pool_once(state, ¤t_pool_key, sql, max_rows).await; let Some(error) = result.as_ref().err() else { return result; }; match transfer_pool_error_action(safety, db_type, error) { PoolErrorAction::Keep => return result, PoolErrorAction::Discard => { state.remove_pool_by_key(¤t_pool_key).await; return result; } PoolErrorAction::ReconnectAndRetry if attempt == 0 => { let Some(connection_id) = connection_id.as_deref() else { state.remove_pool_by_key(¤t_pool_key).await; return result; }; let catalog = catalog_from_pool_key(¤t_pool_key).map(str::to_string); current_pool_key = state .reconnect_pool_for_session_with_catalog( connection_id, database.as_deref(), catalog.as_deref(), client_session_id.as_deref(), ) .await?; } PoolErrorAction::ReconnectAndRetry => { state.remove_pool_by_key(¤t_pool_key).await; return result; } } } unreachable!("transfer pool execution retry loop runs at most twice") } async fn execute_on_pool_once( state: &AppState, pool_key: &str, sql: &str, max_rows: Option, ) -> Result { // Read-only check: block transfer operations in readonly mode crate::query::check_read_only_for_connection(state, pool_key, sql).await?; let pool_handle = state.pool_handle(pool_key).await; let pool = pool_handle.as_ref().ok_or("Connection not found")?; match pool { PoolKind::Mysql(p, mode) => { let p = p.clone(); let bare = *mode == crate::connection::MysqlMode::Bare; db::mysql::execute_query_with_max_rows(&p, sql, bare, max_rows, Default::default()).await } PoolKind::Postgres(p) => { let p = p.clone(); db::postgres::execute_query_with_max_rows(&p, sql, max_rows).await } PoolKind::Sqlite(p) => { let p = p.clone(); db::sqlite::execute_query_with_max_rows(&p, sql, max_rows).await } PoolKind::ClickHouse(client) => { let client = client.clone(); let database = database_from_pool_key(pool_key).unwrap_or("default").to_string(); db::clickhouse_driver::execute_query_with_max_rows(&client, &database, sql, max_rows).await } PoolKind::SqlServer(client) => { let client = client.clone(); let mut client = client.lock().await; let result = db::sqlserver::execute_query_with_max_rows(&mut client, sql, max_rows).await; drop(client); result } PoolKind::Agent(client) => { let client = client.clone(); let database = database_from_pool_key(pool_key).map(str::to_string); let sql = sql.to_string(); let mut client = client.lock().await; let params = agent_execute_query_params( &sql, database.as_deref(), None, QueryExecutionOptions { max_rows, fetch_size: max_rows, ..QueryExecutionOptions::default() }, ); client.execute_query(params).await } PoolKind::ExternalDriver { config, session, .. } => { let database = database_from_pool_key(pool_key) .map(str::to_string) .unwrap_or_else(|| config.effective_database().unwrap_or("").to_string()); let params = crate::query::external_driver_query_params( config.as_ref(), sql, &database, None, &QueryExecutionOptions { max_rows, fetch_size: max_rows, ..QueryExecutionOptions::default() }, ); session.invoke_with_timeout("executeQuery", params, None).await } #[cfg(feature = "duckdb-sidecar")] PoolKind::DuckDbWorker(client) => { let client = client.clone(); let sql = sql.to_string(); client.execute(None, sql, max_rows, None, None).await } _ => Err("Unsupported database type for transfer".to_string()), } } fn database_from_pool_key(pool_key: &str) -> Option<&str> { let base = pool_key.split_once(":session:").map(|(base, _)| base).unwrap_or(pool_key); let base = base.split_once(":catalog:").map(|(base, _)| base).unwrap_or(base); base.split_once(':').map(|(_, database)| database).filter(|database| !database.is_empty()) } fn catalog_from_pool_key(pool_key: &str) -> Option<&str> { let base = pool_key.split_once(":session:").map(|(base, _)| base).unwrap_or(pool_key); base.split_once(":catalog:").map(|(_, catalog)| catalog).filter(|catalog| !catalog.is_empty()) } pub async fn get_db_type(state: &AppState, connection_id: &str) -> Result { let configs = state.configs.read().await; configs .get(connection_id) .map(effective_transfer_database_type) .ok_or_else(|| format!("Connection config not found: {connection_id}")) } fn effective_transfer_database_type(config: &ConnectionConfig) -> DatabaseType { if config.db_type != DatabaseType::Jdbc { return config.db_type; } if config.driver_profile.as_deref().is_some_and(|profile| profile.eq_ignore_ascii_case("gbase8s")) { return DatabaseType::Jdbc; } let jdbc_identity = [ config.driver_profile.as_deref().unwrap_or(""), config.connection_string.as_deref().unwrap_or(""), config.jdbc_driver_class.as_deref().unwrap_or(""), ] .join("\n") .to_ascii_lowercase(); // GBase 8a only: 8s (`gbasedbt`/`jdbc:gbasedbt-sqli`) is Informix-based and // must not be mapped into the MySQL transfer family. let is_gbase_8a = jdbc_identity.contains("jdbc:gbase:") || jdbc_identity.contains("cn.gbase.") || (jdbc_identity.contains("gbase") && !jdbc_identity.contains("gbasedbt") && !jdbc_identity.contains("gbase8s")); if is_gbase_8a { DatabaseType::Gbase } else { DatabaseType::Jdbc } } pub async fn get_columns_for_transfer( state: &AppState, pool_key: &str, _connection_id: &str, database: &str, schema: &str, table: &str, catalog: Option<&str>, ) -> Result, String> { let pool_handle = state.pool_handle(pool_key).await; #[cfg(feature = "duckdb-sidecar")] if let Some(PoolKind::DuckDbWorker(client)) = pool_handle.as_ref() { let client = client.clone(); let database = database.to_string(); let schema = schema.to_string(); let table = table.to_string(); return client.list_columns(database, schema, table).await; } if let Some(PoolKind::ClickHouse(client)) = pool_handle.as_ref() { let client = client.clone(); let database = database.to_string(); let table = table.to_string(); return db::clickhouse_driver::get_columns(&client, &database, &table).await; } if let Some(PoolKind::SqlServer(client)) = pool_handle.as_ref() { let client = client.clone(); let schema = schema.to_string(); let table = table.to_string(); let mut client = client.lock().await; return db::sqlserver::get_columns(&mut client, &schema, &table).await; } if let Some(PoolKind::InfluxDb(client)) = pool_handle.as_ref() { let client = client.clone(); let database = database.to_string(); let table = table.to_string(); return db::influxdb_driver::get_columns(&client, &database, &table).await; } if let Some(PoolKind::InfluxDb3(client)) = pool_handle.as_ref() { let client = client.clone(); let database = database.to_string(); let table = table.to_string(); return db::influxdb3_driver::get_columns(&client, &database, &table).await; } if let Some(PoolKind::Agent(client)) = pool_handle.as_ref() { let client = client.clone(); let database = database.to_string(); let schema = schema.to_string(); let table = table.to_string(); let mut client = client.lock().await; return client.get_columns(&database, &schema, &table, None).await; } if let Some(PoolKind::ExternalDriver { config, session, .. }) = pool_handle.as_ref() { let config = config.clone(); let session = session.clone(); let database = database.to_string(); let schema = schema.to_string(); let table = table.to_string(); return session .invoke_with_timeout( "getColumns", serde_json::json!({ "connection": config.as_ref(), "database": database, "schema": schema, "table": table, }), None, ) .await; } let pool = pool_handle.as_ref().ok_or("Pool not found")?; let schema = schema.to_string(); let table = table.to_string(); match pool { PoolKind::Mysql(p, _) => { let p = p.clone(); let catalog = normalize_external_catalog_name(catalog).map(str::to_string); if let Some(catalog) = catalog { // Use 3-part qualified column lookup for Doris/StarRocks external catalogs db::doris::get_catalog_columns(&p, &catalog, &schema, &table).await } else { db::mysql::get_columns(&p, &schema, &table).await } } PoolKind::Postgres(p) => { let p = p.clone(); db::postgres::get_columns(&p, &schema, &table).await } PoolKind::Sqlite(p) => { let p = p.clone(); db::sqlite::get_columns(&p, &schema, &table).await } _ => Err("Unsupported database type".to_string()), } } async fn get_postgres_indexes_for_transfer( state: &AppState, pool_key: &str, database: &str, schema: &str, table: &str, ) -> Result, String> { let pool_handle = state.pool_handle(pool_key).await; if let Some(PoolKind::Agent(client)) = pool_handle.as_ref() { let client = client.clone(); let database = database.to_string(); let schema = schema.to_string(); let table = table.to_string(); let mut client = client.lock().await; return client.list_indexes(&database, &schema, &table, None).await; } let Some(PoolKind::Postgres(pool)) = pool_handle.as_ref() else { return Err("PostgreSQL pool not found".to_string()); }; let pool = pool.clone(); db::postgres::list_indexes(&pool, schema, table).await } async fn get_postgres_foreign_keys_for_transfer( state: &AppState, pool_key: &str, database: &str, schema: &str, table: &str, ) -> Result, String> { let pool_handle = state.pool_handle(pool_key).await; if let Some(PoolKind::Agent(client)) = pool_handle.as_ref() { let client = client.clone(); let database = database.to_string(); let schema = schema.to_string(); let table = table.to_string(); let mut client = client.lock().await; return client.list_foreign_keys(&database, &schema, &table, None).await; } let Some(PoolKind::Postgres(pool)) = pool_handle.as_ref() else { return Err("PostgreSQL pool not found".to_string()); }; let pool = pool.clone(); db::postgres::list_foreign_keys(&pool, schema, table).await } async fn get_postgres_owned_sequences_for_transfer( state: &AppState, pool_key: &str, schema: &str, tables: &[String], ) -> Result, String> { if tables.is_empty() { return Ok(Vec::new()); } let pool = { let pool_handle = state.pool_handle(pool_key).await; match pool_handle.as_ref() { Some(PoolKind::Postgres(pool)) => pool.clone(), _ => return Ok(Vec::new()), } }; let client = pool.get().await.map_err(|e| e.to_string())?; let rows = client.query(POSTGRES_OWNED_SEQUENCES_SQL, &[&schema]).await.map_err(|e| e.to_string())?; let selected: HashSet<&str> = tables.iter().map(String::as_str).collect(); Ok(rows .iter() .filter_map(|row| { let owner_table = row.get::<_, String>(1); if !selected.contains(owner_table.as_str()) { return None; } Some(PostgresOwnedSequence { name: row.get::<_, String>(0), owner_table, owner_column: row.get::<_, String>(2), }) }) .collect()) } const POSTGRES_OWNED_SEQUENCES_SQL: &str = "SELECT c.relname, \ t.relname, \ a.attname \ FROM pg_class c \ JOIN pg_namespace n ON n.oid = c.relnamespace \ JOIN pg_depend d ON d.classid = 'pg_class'::regclass \ AND d.objid = c.oid \ AND d.refclassid = 'pg_class'::regclass \ AND d.deptype IN ('a', 'i') \ JOIN pg_class t ON t.oid = d.refobjid \ JOIN pg_namespace tn ON tn.oid = t.relnamespace AND tn.nspname = n.nspname \ JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = d.refobjsubid \ WHERE c.relkind = 'S' AND n.nspname = $1 \ ORDER BY t.relname, c.relname"; async fn get_postgres_sequence_snapshots_for_transfer( state: &AppState, pool_key: &str, schema: &str, ) -> Result, String> { let pool = { let pool_handle = state.pool_handle(pool_key).await; match pool_handle.as_ref() { Some(PoolKind::Postgres(pool)) => pool.clone(), _ => return Ok(Vec::new()), } }; let client = pool.get().await.map_err(|e| e.to_string())?; let rows = client.query(POSTGRES_SEQUENCE_SNAPSHOTS_SQL, &[&schema]).await.map_err(|e| e.to_string())?; Ok(rows .iter() .map(|row| PostgresSequenceSnapshot { name: row.get::<_, String>(0), owner_table: row.get::<_, Option>(1), owner_column: row.get::<_, Option>(2), }) .collect()) } const POSTGRES_SEQUENCE_SNAPSHOTS_SQL: &str = "SELECT c.relname, \ t.relname, \ a.attname \ FROM pg_class c \ JOIN pg_namespace n ON n.oid = c.relnamespace \ LEFT JOIN pg_depend d ON d.classid = 'pg_class'::regclass \ AND d.objid = c.oid \ AND d.refclassid = 'pg_class'::regclass \ AND d.deptype IN ('a', 'i') \ LEFT JOIN pg_class t ON t.oid = d.refobjid \ LEFT JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = d.refobjsubid \ WHERE c.relkind = 'S' AND n.nspname = $1 \ ORDER BY c.relname"; fn postgres_selected_sequences_sql(schema: &str, names: &[String]) -> Option { if names.is_empty() { return None; } let name_list = names.iter().map(|name| quote_string_literal(name)).collect::>().join(", "); Some(format!( "SELECT c.relname, \ COALESCE(format_type(s.seqtypid, NULL), 'bigint'), \ COALESCE(s.seqstart::text, '1'), \ COALESCE(s.seqmin::text, '1'), \ COALESCE(s.seqmax::text, '9223372036854775807'), \ COALESCE(s.seqincrement::text, '1'), \ CASE WHEN COALESCE(s.seqcycle, false) THEN 'true' ELSE 'false' END, \ COALESCE(s.seqcache::text, '1'), \ pg_sequence_last_value(c.oid)::text \ FROM pg_catalog.pg_class c \ JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ LEFT JOIN pg_catalog.pg_sequence s ON s.seqrelid = c.oid \ WHERE c.relkind = 'S' AND n.nspname = {} AND c.relname IN ({name_list}) \ ORDER BY c.relname", quote_string_literal(schema) )) } async fn get_postgres_selected_sequences_for_transfer( state: &AppState, pool_key: &str, schema: &str, names: &[String], ) -> Result, String> { let Some(sql) = postgres_selected_sequences_sql(schema, names) else { return Ok(Vec::new()); }; Ok(execute_on_pool(state, pool_key, &sql) .await? .rows .into_iter() .filter_map(|row| { Some(PostgresTransferSequence { name: json_string_cell(&row, 0)?, data_type: json_string_cell(&row, 1)?, start_value: json_string_cell(&row, 2)?, min_value: json_string_cell(&row, 3)?, max_value: json_string_cell(&row, 4)?, increment: json_string_cell(&row, 5)?, cycle: json_string_cell(&row, 6).as_deref() == Some("true"), cache_value: json_string_cell(&row, 7)?, last_value: json_string_cell(&row, 8), }) }) .collect()) } async fn get_existing_postgres_sequence_names_for_transfer( state: &AppState, pool_key: &str, schema: &str, names: &[String], ) -> Result, String> { if names.is_empty() { return Ok(HashSet::new()); } let name_list = names.iter().map(|name| quote_string_literal(name)).collect::>().join(", "); let sql = format!( "SELECT c.relname \ FROM pg_catalog.pg_class c \ JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ WHERE c.relkind = 'S' AND n.nspname = {} AND c.relname IN ({name_list})", quote_string_literal(schema) ); Ok(execute_on_pool(state, pool_key, &sql) .await? .rows .into_iter() .filter_map(|row| json_string_cell(&row, 0)) .collect()) } /// Create owned PostgreSQL sequences before executing reused table DDL because /// serial defaults still reference `nextval('...')` in `CREATE TABLE`. async fn prepare_postgres_owned_sequences_for_transfer( state: &AppState, request: &TransferRequest, table: &str, target_table: &str, source_pool_key: &str, target_pool_key: &str, pg_compat_transfer: bool, preserves_target_table_name: bool, target_table_preexisting: bool, ) -> Result, String> { if !(request.create_table && pg_compat_transfer && preserves_target_table_name && !target_table_preexisting) { return Ok(Vec::new()); } let owned_sequences = get_postgres_owned_sequences_for_transfer(state, source_pool_key, &request.source_schema, &[table.to_string()]) .await?; if owned_sequences.is_empty() { return Ok(Vec::new()); } let existing_sequences = get_postgres_sequence_snapshots_for_transfer(state, target_pool_key, &request.target_schema) .await? .into_iter() .map(|sequence| (sequence.name.clone(), sequence)) .collect::>(); for sequence in &owned_sequences { let should_create = validate_existing_postgres_sequence( sequence, existing_sequences.get(&sequence.name), &request.target_schema, )?; if should_create { let create_sql = format!("CREATE SEQUENCE {}", postgres_sequence_qualified_name(&request.target_schema, &sequence.name)); execute_on_pool(state, target_pool_key, &create_sql) .await .map_err(|e| format!("Failed to create PostgreSQL sequence for {target_table}: {e}"))?; } } Ok(owned_sequences) } /// Bind created or reused sequences after the table exists so /// `pg_get_serial_sequence(...)` can find them during later sequence sync. async fn bind_postgres_owned_sequences_for_transfer( state: &AppState, request: &TransferRequest, target_table: &str, target_pool_key: &str, owned_sequences: &[PostgresOwnedSequence], ) -> Result<(), String> { for sequence in owned_sequences { let owner_sql = format!( "ALTER SEQUENCE {} OWNED BY {}.{}", postgres_sequence_qualified_name(&request.target_schema, &sequence.name), qualified_table(&sequence.owner_table, &request.target_schema, &DatabaseType::Postgres, None), quote_identifier(&sequence.owner_column, &DatabaseType::Postgres) ); execute_on_pool(state, target_pool_key, &owner_sql) .await .map_err(|e| format!("Failed to bind PostgreSQL sequence for {target_table}: {e}"))?; } Ok(()) } pub fn ordered_transfer_object_kinds(kinds: Vec) -> Vec { let rank = |kind: &TransferObjectKind| match kind { TransferObjectKind::Table => 0, TransferObjectKind::Sequence => 1, TransferObjectKind::View => 2, TransferObjectKind::MaterializedView => 2, TransferObjectKind::Function => 3, TransferObjectKind::Procedure => 4, TransferObjectKind::Trigger => 5, TransferObjectKind::Event => 6, }; let mut kinds = kinds; kinds.sort_by_key(rank); kinds } #[derive(Debug, Clone, Default, Serialize)] #[serde(rename_all = "camelCase")] pub struct TransferObjectOutcome { pub transferred: Vec, pub skipped: Vec, pub failed: Vec, } pub fn selected_object_names(selections: &[TransferObjectSelection], kind: &TransferObjectKind) -> Vec { selections.iter().filter(|s| &s.object_type == kind).flat_map(|s| s.names.clone()).collect::>() } fn selected_postgres_sequence_names(request: &TransferRequest) -> Vec { let mut names = selected_object_names(&request.objects, &TransferObjectKind::Sequence); names.sort(); names.dedup(); names } fn postgres_transfer_relation_names(request: &TransferRequest) -> Vec { let mut names = request.tables.clone(); names.extend(selected_postgres_sequence_names(request)); names.sort(); names.dedup(); names } /// Whether a kind participates in a transfer. An empty selection is the legacy /// PG→PG default: every kind participates (views, functions, triggers, /// materialized views are all transferred). Once the caller explicitly selects /// objects, only kinds with a non-empty selection participate. pub fn object_kind_selected_or_defaulted(selections: &[TransferObjectSelection], kind: &TransferObjectKind) -> bool { selections.is_empty() || !selected_object_names(selections, kind).is_empty() } pub fn should_copy_data(content: &TransferContent) -> bool { !matches!(content, TransferContent::StructureOnly) } fn transfer_kind_from_object_source_kind(kind: &db::ObjectSourceKind) -> Option { use db::ObjectSourceKind as S; Some(match kind { S::View => TransferObjectKind::View, S::MaterializedView => TransferObjectKind::MaterializedView, S::Procedure => TransferObjectKind::Procedure, S::Function => TransferObjectKind::Function, S::Trigger => TransferObjectKind::Trigger, S::Sequence => TransferObjectKind::Sequence, _ => return None, }) } fn filter_object_sources_by_selection( sources: Vec, selections: &[TransferObjectSelection], ) -> Vec { // Empty selection is the legacy PG→PG default: transfer everything. if selections.is_empty() { return sources; } sources .into_iter() .filter(|source| { let Some(kind) = transfer_kind_from_object_source_kind(&source.object_type) else { return false; }; selected_object_names(selections, &kind).contains(&source.name) }) .collect() } /// Whether non-table schema-object transfer should run for a request. /// Data-only transfers never include schema objects. In structure modes, /// newer clients send an explicit `objects` selection; for those the answer /// is simply whether any object was selected. PG→PG keeps the legacy default: /// even an *empty* selection still transfers all views, functions, triggers, /// policies, ownership and grants (the old table-selection flow always did). pub fn should_transfer_schema_objects( source_db_type: &DatabaseType, target_db_type: &DatabaseType, content: &TransferContent, objects: &[TransferObjectSelection], ) -> bool { if matches!(content, TransferContent::DataOnly) { return false; } if !objects.is_empty() { return true; } transfer_object_family(source_db_type) == Some(TransferObjectFamily::Postgres) && transfer_object_family(target_db_type) == Some(TransferObjectFamily::Postgres) } /// Transfers selected non-table objects from source to target. /// Skips objects that already exist on the target; counts them in the /// outcome. Executes in dependency order (sequence → view → function → /// procedure → trigger → event). Errors are collected per object and the /// transfer continues. pub async fn transfer_schema_objects( state: &AppState, request: &TransferRequest, source_pool_key: &str, target_pool_key: &str, progress_callback: F, ) -> Result where F: FnMut(TransferProgress), { if matches!(request.content, TransferContent::DataOnly) { return Ok(TransferObjectOutcome::default()); } let source_db_type = get_db_type(state, &request.source_connection_id).await?; let target_db_type = get_db_type(state, &request.target_connection_id).await?; if !should_transfer_schema_objects(&source_db_type, &target_db_type, &request.content, &request.objects) { return Ok(TransferObjectOutcome::default()); } if !is_same_transfer_family(&source_db_type, &target_db_type) { return transfer_cross_family_schema_objects( state, request, source_pool_key, target_pool_key, progress_callback, ) .await; } // Same-family path: drop selections whose object type is not transferable // for the source family (defense in depth — the UI already filters disabled // types at the request boundary, but requests can also arrive from older // clients or be crafted directly). let mut filtered_request = request.clone(); if let Some(family) = transfer_object_family(&source_db_type) { let supported = transfer_object_kinds_for_family(&family); filtered_request.objects = request.objects.iter().filter(|sel| supported.contains(&sel.object_type)).cloned().collect(); } match transfer_object_family(&source_db_type) { Some(TransferObjectFamily::Postgres) => { transfer_postgres_schema_objects( state, &filtered_request, source_pool_key, target_pool_key, progress_callback, ) .await } Some(TransferObjectFamily::Mysql) => { transfer_mysql_schema_objects(state, &filtered_request, source_pool_key, target_pool_key, progress_callback) .await } Some(TransferObjectFamily::Oracle) => { transfer_oracle_schema_objects( state, &filtered_request, source_pool_key, target_pool_key, progress_callback, ) .await } Some(TransferObjectFamily::SqlServer) => { transfer_sqlserver_schema_objects( state, &filtered_request, source_pool_key, target_pool_key, progress_callback, ) .await } None => Ok(TransferObjectOutcome::default()), } } async fn transfer_mysql_schema_objects( state: &AppState, request: &TransferRequest, source_pool_key: &str, target_pool_key: &str, mut progress_callback: F, ) -> Result where F: FnMut(TransferProgress), { let mut outcome = TransferObjectOutcome::default(); let source_db = &request.source_database; let target_db = if request.target_database.trim().is_empty() { source_db.as_str() } else { request.target_database.as_str() }; let order = ordered_transfer_object_kinds(request.objects.iter().map(|s| s.object_type).collect()); for kind in order { for name in selected_object_names(&request.objects, &kind) { if is_cancelled(&request.transfer_id).await { return Err("Cancelled".to_string()); } let table = format!("schema object: {name}"); let mut progress = |outcome: &mut TransferObjectOutcome, status: TransferStatus, error: Option| { progress_callback(TransferProgress { transfer_id: request.transfer_id.clone(), table: table.clone(), table_index: request.tables.len(), total_tables: request.tables.len(), rows_transferred: (outcome.transferred.len() + outcome.skipped.len()) as u64, total_rows: None, status, error, terminal: false, }); }; // skip if the target already has it let exists_sql = target_object_exists_sql(&DatabaseType::Mysql, target_db, &name, &kind)?; let exists = !execute_on_pool(state, target_pool_key, &exists_sql).await?.rows.is_empty(); if exists { outcome.skipped.push(format!("{kind:?}:{name}")); progress(&mut outcome, TransferStatus::Running, None); continue; } let query = mysql_object_source_query(&kind, source_db, &name)?; let result = execute_on_pool(state, source_pool_key, &query).await?; let raw_ddl = mysql_object_ddl_from_result(&kind, source_db, &result.rows)?; let ddl = strip_mysql_definer(&raw_ddl); let ddl = rewrite_mysql_schema_qualifier(&ddl, source_db, target_db); match execute_on_pool(state, target_pool_key, &ddl).await { Ok(_) => { outcome.transferred.push(format!("{kind:?}:{name}")); progress(&mut outcome, TransferStatus::Running, None); } Err(e) => { outcome.failed.push(format!("{kind:?}:{name}")); progress(&mut outcome, TransferStatus::Error, Some(e)); } } } } Ok(outcome) } fn resolve_oracle_schema(schema: &str, database: &str) -> String { if schema.trim().is_empty() { database.to_string() } else { schema.to_string() } } async fn transfer_oracle_schema_objects( state: &AppState, request: &TransferRequest, source_pool_key: &str, target_pool_key: &str, mut progress_callback: F, ) -> Result where F: FnMut(TransferProgress), { let mut outcome = TransferObjectOutcome::default(); let source_schema = resolve_oracle_schema(&request.source_schema, &request.source_database); let target_schema = resolve_oracle_schema(&request.target_schema, &request.target_database); let order = ordered_transfer_object_kinds(request.objects.iter().map(|s| s.object_type).collect()); for kind in order { for name in selected_object_names(&request.objects, &kind) { if is_cancelled(&request.transfer_id).await { return Err("Cancelled".to_string()); } let table = format!("schema object: {name}"); let mut progress = |outcome: &mut TransferObjectOutcome, status: TransferStatus, error: Option| { progress_callback(TransferProgress { transfer_id: request.transfer_id.clone(), table: table.clone(), table_index: request.tables.len(), total_tables: request.tables.len(), rows_transferred: (outcome.transferred.len() + outcome.skipped.len()) as u64, total_rows: None, status, error, terminal: false, }); }; // skip if the target already has it (ALL_OBJECTS works for both Oracle and Dameng) let exists_sql = target_object_exists_sql(&DatabaseType::Oracle, &target_schema, &name, &kind)?; let exists = !execute_on_pool(state, target_pool_key, &exists_sql).await?.rows.is_empty(); if exists { outcome.skipped.push(format!("{kind:?}:{name}")); progress(&mut outcome, TransferStatus::Running, None); continue; } let query = oracle_object_source_query(&kind, &source_schema, &name)?; let result = execute_on_pool(state, source_pool_key, &query).await?; // DBMS_METADATA.GET_DDL resolves against the session's current // schema; if the pool session is not the owner schema the query // may return no row and the object is reported as failed (v1). let ddl = result .rows .first() .and_then(|row| row.first()) .and_then(|value| value.as_str()) .ok_or_else(|| format!("No DDL returned for Oracle {:?} {name}", kind))? .to_string(); let ddl = rewrite_oracle_schema_qualifier(&ddl, &source_schema, &target_schema); match execute_on_pool(state, target_pool_key, &ddl).await { Ok(_) => { outcome.transferred.push(format!("{kind:?}:{name}")); progress(&mut outcome, TransferStatus::Running, None); } Err(e) => { outcome.failed.push(format!("{kind:?}:{name}")); progress(&mut outcome, TransferStatus::Error, Some(e)); } } } } Ok(outcome) } async fn transfer_sqlserver_schema_objects( state: &AppState, request: &TransferRequest, source_pool_key: &str, target_pool_key: &str, mut progress_callback: F, ) -> Result where F: FnMut(TransferProgress), { let mut outcome = TransferObjectOutcome::default(); let source_schema = if request.source_schema.trim().is_empty() { "dbo".to_string() } else { request.source_schema.clone() }; let target_schema = if request.target_schema.trim().is_empty() { "dbo".to_string() } else { request.target_schema.clone() }; let order = ordered_transfer_object_kinds(request.objects.iter().map(|s| s.object_type).collect()); for kind in order { for name in selected_object_names(&request.objects, &kind) { if is_cancelled(&request.transfer_id).await { return Err("Cancelled".to_string()); } let table = format!("schema object: {name}"); let mut progress = |outcome: &mut TransferObjectOutcome, status: TransferStatus, error: Option| { progress_callback(TransferProgress { transfer_id: request.transfer_id.clone(), table: table.clone(), table_index: request.tables.len(), total_tables: request.tables.len(), rows_transferred: (outcome.transferred.len() + outcome.skipped.len()) as u64, total_rows: None, status, error, terminal: false, }); }; let exists_sql = target_object_exists_sql(&DatabaseType::SqlServer, &target_schema, &name, &kind)?; let exists = !execute_on_pool(state, target_pool_key, &exists_sql).await?.rows.is_empty(); if exists { outcome.skipped.push(format!("{kind:?}:{name}")); progress(&mut outcome, TransferStatus::Running, None); continue; } let query = sqlserver_object_source_query(&kind, &source_schema, &name)?; let result = execute_on_pool(state, source_pool_key, &query).await?; let ddl = match sqlserver_object_ddl_from_result(&result, &source_schema, &name, &kind) { Ok(ddl) => ddl, Err(e) => { outcome.failed.push(format!("{kind:?}:{name}")); progress(&mut outcome, TransferStatus::Error, Some(e)); continue; } }; match execute_on_pool(state, target_pool_key, &ddl).await { Ok(_) => { outcome.transferred.push(format!("{kind:?}:{name}")); progress(&mut outcome, TransferStatus::Running, None); } Err(e) => { outcome.failed.push(format!("{kind:?}:{name}")); progress(&mut outcome, TransferStatus::Error, Some(e)); } } } } Ok(outcome) } /// Transfers non-table objects across different database families. /// Only mechanically rewriteable kinds (views, sequences) are allowed; /// anything else is rejected up front with a descriptive error. async fn transfer_cross_family_schema_objects( state: &AppState, request: &TransferRequest, source_pool_key: &str, target_pool_key: &str, mut progress_callback: F, ) -> Result where F: FnMut(TransferProgress), { let mut outcome = TransferObjectOutcome::default(); let source_db_type = get_db_type(state, &request.source_connection_id).await?; let target_db_type = get_db_type(state, &request.target_connection_id).await?; let allowed = cross_family_transferable_object_kinds(&source_db_type, &target_db_type); let unsupported: Vec = request .objects .iter() .filter(|selection| !allowed.contains(&selection.object_type)) .map(|selection| format!("{:?}", selection.object_type)) .collect(); if !unsupported.is_empty() { return Err(format!("跨库非表对象传输暂不支持该类型,不支持: {}", unsupported.join(", "))); } let source_family = transfer_object_family(&source_db_type).ok_or("unsupported source family")?; let target_family = transfer_object_family(&target_db_type).ok_or("unsupported target family")?; let resolve_schema = |schema: &str, database: &str, db_type: &DatabaseType| -> String { if !schema.trim().is_empty() { return schema.to_string(); } match transfer_object_family(db_type) { Some(TransferObjectFamily::SqlServer) => "dbo".to_string(), _ => database.to_string(), } }; let source_schema = resolve_schema(&request.source_schema, &request.source_database, &source_db_type); let target_schema = resolve_schema(&request.target_schema, &request.target_database, &target_db_type); let order = ordered_transfer_object_kinds(request.objects.iter().map(|s| s.object_type).collect()); for kind in order { for name in selected_object_names(&request.objects, &kind) { if is_cancelled(&request.transfer_id).await { return Err("Cancelled".to_string()); } let table = format!("schema object: {name}"); let mut progress = |outcome: &mut TransferObjectOutcome, status: TransferStatus, error: Option| { progress_callback(TransferProgress { transfer_id: request.transfer_id.clone(), table: table.clone(), table_index: request.tables.len(), total_tables: request.tables.len(), rows_transferred: (outcome.transferred.len() + outcome.skipped.len()) as u64, total_rows: None, status, error, terminal: false, }); }; let exists_sql = target_object_exists_sql(&target_db_type, &target_schema, &name, &kind)?; let exists = !execute_on_pool(state, target_pool_key, &exists_sql).await?.rows.is_empty(); if exists { outcome.skipped.push(format!("{kind:?}:{name}")); progress(&mut outcome, TransferStatus::Running, None); continue; } let query = match source_family { TransferObjectFamily::Mysql => mysql_object_source_query(&kind, &source_schema, &name)?, TransferObjectFamily::SqlServer => sqlserver_object_source_query(&kind, &source_schema, &name)?, TransferObjectFamily::Oracle => oracle_object_source_query(&kind, &source_schema, &name)?, TransferObjectFamily::Postgres => { return Err(format!("跨库传输暂不支持 Postgres 源: {name}")); } }; let result = execute_on_pool(state, source_pool_key, &query).await?; let raw_ddl = match source_family { TransferObjectFamily::Mysql => mysql_object_ddl_from_result(&kind, &source_schema, &result.rows)?, TransferObjectFamily::SqlServer => { sqlserver_object_ddl_from_result(&result, &source_schema, &name, &kind)? } TransferObjectFamily::Oracle => result .rows .first() .and_then(|row| row.first()) .and_then(|value| value.as_str()) .ok_or_else(|| format!("No DDL returned for Oracle {:?} {name}", kind))? .to_string(), TransferObjectFamily::Postgres => { return Err(format!("跨库传输暂不支持 Postgres 源: {name}")); } }; let ddl = convert_cross_family_object_ddl( &source_family, &target_family, &kind, &source_schema, &target_schema, &raw_ddl, ); match execute_on_pool(state, target_pool_key, &ddl).await { Ok(_) => { outcome.transferred.push(format!("{kind:?}:{name}")); progress(&mut outcome, TransferStatus::Running, None); } Err(e) => { let e = if kind == TransferObjectKind::View && (e.contains("无效的表或视图名") || e.contains("table or view does not exist")) { format!("{e}(视图引用的基表可能未在目标库中,请同时选择视图依赖的表或先传输这些表)") } else { e }; outcome.failed.push(format!("{kind:?}:{name}")); progress(&mut outcome, TransferStatus::Error, Some(e)); } } } } Ok(outcome) } fn postgres_transfer_catalog_capabilities_sql() -> &'static str { "SELECT EXISTS ( \ SELECT 1 \ FROM pg_catalog.pg_attribute \ WHERE attrelid = 'pg_catalog.pg_proc'::regclass \ AND attname = 'prokind' \ AND NOT attisdropped \ ), EXISTS ( \ SELECT 1 \ FROM pg_catalog.pg_class c \ JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ WHERE n.nspname = 'pg_catalog' AND c.relname = 'pg_policy' \ ), EXISTS ( \ SELECT 1 \ FROM pg_catalog.pg_attribute a \ JOIN pg_catalog.pg_class c ON c.oid = a.attrelid \ JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ WHERE n.nspname = 'pg_catalog' \ AND c.relname = 'pg_policy' \ AND a.attname = 'polpermissive' \ AND NOT a.attisdropped \ )" } struct PostgresTransferCatalogCapabilities { has_prokind: bool, has_pg_policy: bool, supports_policy_permissiveness: bool, } fn postgres_capability_cell(row: &[serde_json::Value], index: usize) -> Option { row.get(index).and_then(|value| value.as_bool().or_else(|| value.as_str().and_then(|value| value.parse().ok()))) } async fn postgres_transfer_catalog_capabilities( state: &AppState, pool_key: &str, ) -> Result { let result = execute_read_on_pool(state, pool_key, postgres_transfer_catalog_capabilities_sql()).await?; let row = result.rows.first().ok_or_else(|| "Failed to inspect PostgreSQL transfer catalog capabilities".to_string())?; Ok(PostgresTransferCatalogCapabilities { has_prokind: postgres_capability_cell(row, 0) .ok_or_else(|| "Failed to inspect PostgreSQL routine catalog capabilities".to_string())?, has_pg_policy: postgres_capability_cell(row, 1) .ok_or_else(|| "Failed to inspect PostgreSQL policy catalog capabilities".to_string())?, supports_policy_permissiveness: postgres_capability_cell(row, 2) .ok_or_else(|| "Failed to inspect PostgreSQL policy catalog capabilities".to_string())?, }) } fn postgres_transfer_routine_catalog_sql(has_prokind: bool) -> (&'static str, &'static str) { if has_prokind { ("CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END", "p.prokind IN ('p','f')") } else { ("'FUNCTION'::text", "NOT p.proisagg AND NOT p.proiswindow") } } fn postgres_transfer_routines_sql(schema: &str, has_prokind: bool) -> String { let (routine_kind, routine_filter) = postgres_transfer_routine_catalog_sql(has_prokind); format!( "SELECT p.proname, {routine_kind}, pg_get_functiondef(p.oid) \ FROM pg_catalog.pg_proc p \ JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace \ WHERE n.nspname = {schema} AND {routine_filter} \ ORDER BY CASE WHEN {routine_kind} = 'PROCEDURE' THEN 0 ELSE 1 END, p.proname, p.oid", schema = quote_string_literal(schema), ) } async fn get_postgres_schema_object_sources_for_transfer( state: &AppState, pool_key: &str, schema: &str, has_prokind: bool, ) -> Result, String> { let views_sql = format!( "SELECT c.relname, pg_get_viewdef(c.oid, true) \ FROM pg_catalog.pg_class c \ JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ WHERE n.nspname = {} AND c.relkind = 'v' \ ORDER BY c.relname", quote_string_literal(schema) ); let routines_sql = postgres_transfer_routines_sql(schema, has_prokind); let mut sources = Vec::new(); for row in execute_on_pool(state, pool_key, &views_sql).await?.rows { let Some(name) = json_string_cell(&row, 0) else { continue; }; let Some(source) = json_string_cell(&row, 1) else { continue; }; sources.push(db::ObjectSource { name, object_type: db::ObjectSourceKind::View, schema: Some(schema.to_string()), source, editable: None, }); } for row in execute_on_pool(state, pool_key, &routines_sql).await?.rows { let Some(name) = json_string_cell(&row, 0) else { continue; }; let kind = match json_string_cell(&row, 1).as_deref() { Some("PROCEDURE") => db::ObjectSourceKind::Procedure, _ => db::ObjectSourceKind::Function, }; let Some(source) = json_string_cell(&row, 2) else { continue; }; sources.push(db::ObjectSource { name, object_type: kind, schema: Some(schema.to_string()), source, editable: None, }); } Ok(sources) } async fn get_postgres_materialized_view_sources_for_transfer( state: &AppState, pool_key: &str, schema: &str, ) -> Result, String> { let sql = format!( "SELECT c.relname, pg_get_viewdef(c.oid, true) \ FROM pg_catalog.pg_class c \ JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ WHERE n.nspname = {} AND c.relkind = 'm' \ ORDER BY c.relname", quote_string_literal(schema) ); let rows = execute_on_pool(state, pool_key, &sql).await?.rows; Ok(rows .into_iter() .filter_map(|row| { Some(PostgresMaterializedViewSource { view_name: json_string_cell(&row, 0)?, source: json_string_cell(&row, 1)?, }) }) .collect()) } async fn get_postgres_trigger_sources_for_transfer( state: &AppState, pool_key: &str, schema: &str, tables: &[String], ) -> Result, String> { if tables.is_empty() { return Ok(Vec::new()); } let table_list = tables.iter().map(|table| quote_string_literal(table)).collect::>().join(", "); let sql = format!( "SELECT c.relname, t.tgname, pg_get_triggerdef(t.oid, true) \ FROM pg_catalog.pg_trigger t \ JOIN pg_catalog.pg_class c ON c.oid = t.tgrelid \ JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ WHERE n.nspname = {} AND NOT t.tgisinternal AND c.relname IN ({table_list}) \ ORDER BY c.relname, t.tgname", quote_string_literal(schema) ); let rows = execute_on_pool(state, pool_key, &sql).await?.rows; Ok(rows .into_iter() .filter_map(|row| { Some(PostgresTriggerSource { table_name: json_string_cell(&row, 0)?, trigger_name: json_string_cell(&row, 1)?, source: json_string_cell(&row, 2)?, }) }) .collect()) } async fn get_postgres_extension_sources_for_transfer( state: &AppState, pool_key: &str, schema: &str, ) -> Result, String> { let sql = format!( "SELECT e.extname \ FROM pg_extension e \ JOIN pg_namespace n ON n.oid = e.extnamespace \ WHERE n.nspname = {} \ ORDER BY e.extname", quote_string_literal(schema) ); let rows = execute_on_pool(state, pool_key, &sql).await?.rows; Ok(rows .into_iter() .filter_map(|row| json_string_cell(&row, 0).map(|extension_name| PostgresExtensionSource { extension_name })) .collect()) } async fn get_postgres_enum_sources_for_transfer( state: &AppState, pool_key: &str, schema: &str, ) -> Result, String> { let sql = format!( "SELECT t.typname, COALESCE(array_to_json(array_agg(e.enumlabel ORDER BY e.enumsortorder))::text, '[]') \ FROM pg_type t \ JOIN pg_namespace n ON n.oid = t.typnamespace \ LEFT JOIN pg_enum e ON e.enumtypid = t.oid \ WHERE n.nspname = {} AND t.typtype = 'e' \ GROUP BY t.typname \ ORDER BY t.typname", quote_string_literal(schema) ); let rows = execute_on_pool(state, pool_key, &sql).await?.rows; Ok(rows .into_iter() .filter_map(|row| { let type_name = json_string_cell(&row, 0)?; let labels_json = json_string_cell(&row, 1)?; let labels = serde_json::from_str::>(&labels_json).ok()?; Some(PostgresEnumSource { type_name, labels }) }) .collect()) } async fn get_postgres_domain_sources_for_transfer( state: &AppState, pool_key: &str, schema: &str, ) -> Result, String> { let sql = format!( "SELECT t.typname, \ pg_catalog.format_type(t.typbasetype, t.typtypmod), \ NULLIF(t.typdefault, ''), \ t.typnotnull, \ COALESCE(( \ SELECT array_to_json(array_agg(pg_get_constraintdef(c.oid, true) ORDER BY c.conname))::text \ FROM pg_constraint c \ WHERE c.contypid = t.oid AND c.contype = 'c' \ ), '[]') \ FROM pg_type t \ JOIN pg_namespace n ON n.oid = t.typnamespace \ WHERE n.nspname = {} AND t.typtype = 'd' \ ORDER BY t.typname", quote_string_literal(schema) ); let rows = execute_on_pool(state, pool_key, &sql).await?.rows; Ok(rows .into_iter() .filter_map(|row| { let domain_name = json_string_cell(&row, 0)?; let base_type = json_string_cell(&row, 1)?; let default_value = json_string_cell(&row, 2); let not_null = row.get(3).and_then(|value| value.as_bool()).unwrap_or(false); let checks = json_string_cell(&row, 4) .and_then(|json| serde_json::from_str::>(&json).ok()) .unwrap_or_default(); Some(PostgresDomainSource { domain_name, base_type, default_value, not_null, checks }) }) .collect()) } async fn get_postgres_policy_statements_for_transfer( state: &AppState, pool_key: &str, source_schema: &str, target_schema: &str, tables: &[String], has_pg_policy: bool, supports_policy_permissiveness: bool, ) -> Result, String> { if tables.is_empty() || !has_pg_policy { return Ok(Vec::new()); } let (policy_permissiveness_select, policy_permissiveness_clause) = if supports_policy_permissiveness { ("p.polpermissive", "CASE WHEN polpermissive THEN ' AS PERMISSIVE' ELSE ' AS RESTRICTIVE' END") } else { ("true", "''") }; let table_list = tables.iter().map(|table| quote_string_literal(table)).collect::>().join(", "); let sql = format!( "WITH selected_tables AS ( \ SELECT c.oid, c.relname, c.relrowsecurity, c.relforcerowsecurity \ FROM pg_catalog.pg_class c \ JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ WHERE n.nspname = {source_schema} AND c.relkind IN ('r','p') AND c.relname IN ({table_list}) \ ), \ policy_rows AS ( \ SELECT t.relname, t.relrowsecurity, t.relforcerowsecurity, p.polname, \ {policy_permissiveness_select} AS polpermissive, p.polcmd, \ COALESCE((SELECT string_agg(CASE WHEN role_oid = 0 THEN 'PUBLIC' ELSE quote_ident(r.rolname) END, ', ' ORDER BY CASE WHEN role_oid = 0 THEN '' ELSE r.rolname END) \ FROM unnest(p.polroles) AS role_oid LEFT JOIN pg_roles r ON r.oid = role_oid), '') AS role_list, \ pg_get_expr(p.polqual, p.polrelid) AS using_expr, \ pg_get_expr(p.polwithcheck, p.polrelid) AS with_check_expr \ FROM selected_tables t \ JOIN pg_catalog.pg_policy p ON p.polrelid = t.oid \ ) \ SELECT stmt FROM ( \ SELECT format('ALTER TABLE %I.%I ENABLE ROW LEVEL SECURITY', {target_schema}, relname) AS stmt, relname, 0 AS sort_order \ FROM selected_tables WHERE relrowsecurity \ UNION ALL \ SELECT format('ALTER TABLE %I.%I FORCE ROW LEVEL SECURITY', {target_schema}, relname) AS stmt, relname, 1 AS sort_order \ FROM selected_tables WHERE relforcerowsecurity \ UNION ALL \ SELECT format('DROP POLICY IF EXISTS %I ON %I.%I', polname, {target_schema}, relname) AS stmt, relname, 2 AS sort_order \ FROM policy_rows \ UNION ALL \ SELECT format( \ 'CREATE POLICY %I ON %I.%I%s FOR %s%s%s%s', \ polname, {target_schema}, relname, \ {policy_permissiveness_clause}, \ CASE polcmd WHEN 'r' THEN 'SELECT' WHEN 'a' THEN 'INSERT' WHEN 'w' THEN 'UPDATE' WHEN 'd' THEN 'DELETE' ELSE 'ALL' END, \ CASE WHEN role_list <> '' THEN ' TO ' || role_list ELSE '' END, \ CASE WHEN using_expr IS NOT NULL THEN ' USING (' || using_expr || ')' ELSE '' END, \ CASE WHEN with_check_expr IS NOT NULL THEN ' WITH CHECK (' || with_check_expr || ')' ELSE '' END \ ) AS stmt, relname, 3 AS sort_order \ FROM policy_rows \ ) statements \ ORDER BY relname, sort_order, stmt", source_schema = quote_string_literal(source_schema), target_schema = quote_string_literal(target_schema), policy_permissiveness_select = policy_permissiveness_select, policy_permissiveness_clause = policy_permissiveness_clause, ); Ok(result_rows_to_string_statements(execute_on_pool(state, pool_key, &sql).await?.rows)) } fn postgres_transfer_ownership_statements_sql( source_schema: &str, target_schema: &str, tables: &[String], has_prokind: bool, ) -> String { let table_list = tables.iter().map(|table| quote_string_literal(table)).collect::>().join(", "); let table_filter = if tables.is_empty() { "FALSE".to_string() } else { format!("c.relname IN ({table_list})") }; let (routine_kind, routine_filter) = postgres_transfer_routine_catalog_sql(has_prokind); format!( "WITH relation_owners AS ( \ SELECT CASE c.relkind \ WHEN 'm' THEN format('ALTER MATERIALIZED VIEW %I.%I OWNER TO ', {target_schema}, c.relname) \ WHEN 'v' THEN format('ALTER VIEW %I.%I OWNER TO ', {target_schema}, c.relname) \ WHEN 'f' THEN format('ALTER FOREIGN TABLE %I.%I OWNER TO ', {target_schema}, c.relname) \ WHEN 'S' THEN format('ALTER SEQUENCE %I.%I OWNER TO ', {target_schema}, c.relname) \ ELSE format('ALTER TABLE %I.%I OWNER TO ', {target_schema}, c.relname) \ END AS stmt_prefix, \ pg_get_userbyid(c.relowner) AS owner_name \ FROM pg_catalog.pg_class c \ JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ WHERE n.nspname = {source_schema} AND (c.relkind IN ('v','m') OR ({table_filter} AND c.relkind IN ('r','p','f','S'))) \ ), \ routine_owners AS ( \ SELECT format('ALTER %s %I.%I(%s) OWNER TO ', \ {routine_kind}, {target_schema}, p.proname, pg_get_function_identity_arguments(p.oid)) AS stmt_prefix, \ pg_get_userbyid(p.proowner) AS owner_name \ FROM pg_catalog.pg_proc p \ JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace \ WHERE n.nspname = {source_schema} AND {routine_filter} \ ), \ type_owners AS ( \ SELECT format('ALTER %s %I.%I OWNER TO ', \ CASE t.typtype WHEN 'd' THEN 'DOMAIN' ELSE 'TYPE' END, \ {target_schema}, t.typname) AS stmt_prefix, \ pg_get_userbyid(t.typowner) AS owner_name \ FROM pg_catalog.pg_type t \ JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace \ WHERE n.nspname = {source_schema} AND t.typtype IN ('e','d') \ ) \ SELECT stmt_prefix, owner_name FROM ( \ SELECT format('ALTER SCHEMA %I OWNER TO ', {target_schema}) AS stmt_prefix, \ pg_get_userbyid(n.nspowner) AS owner_name \ FROM pg_catalog.pg_namespace n WHERE n.nspname = {source_schema} \ UNION ALL SELECT stmt_prefix, owner_name FROM relation_owners \ UNION ALL SELECT stmt_prefix, owner_name FROM routine_owners \ UNION ALL SELECT stmt_prefix, owner_name FROM type_owners \ ) statements \ WHERE stmt_prefix IS NOT NULL AND owner_name IS NOT NULL", source_schema = quote_string_literal(source_schema), target_schema = quote_string_literal(target_schema), table_filter = table_filter, ) } async fn get_postgres_ownership_statements_for_transfer( state: &AppState, pool_key: &str, source_schema: &str, target_schema: &str, tables: &[String], has_prokind: bool, ) -> Result, String> { let sql = postgres_transfer_ownership_statements_sql(source_schema, target_schema, tables, has_prokind); Ok(result_rows_to_postgres_ownership_statements(execute_on_pool(state, pool_key, &sql).await?.rows)) } fn distinct_postgres_ownership_roles(statements: &[PostgresOwnershipStatement]) -> Vec { let mut roles = statements.iter().map(|statement| statement.owner.clone()).collect::>(); roles.sort(); roles.dedup(); roles } async fn get_postgres_current_user(state: &AppState, target_pool_key: &str) -> Result { let rows = execute_on_pool(state, target_pool_key, "SELECT current_user").await?.rows; rows.first() .and_then(|row| json_string_cell(row, 0)) .filter(|user| !user.trim().is_empty()) .ok_or_else(|| "Failed to read target PostgreSQL current user".to_string()) } async fn get_existing_postgres_roles( state: &AppState, target_pool_key: &str, roles: &[String], ) -> Result, String> { if roles.is_empty() { return Ok(HashSet::new()); } let role_list = roles.iter().map(|role| quote_string_literal(role)).collect::>().join(", "); let sql = format!("SELECT rolname FROM pg_catalog.pg_roles WHERE rolname IN ({role_list})"); let rows = execute_on_pool(state, target_pool_key, &sql).await?.rows; Ok(rows.into_iter().filter_map(|row| json_string_cell(&row, 0)).collect()) } fn build_postgres_ownership_statement(statement: &PostgresOwnershipStatement, owner: &str) -> String { format!("{}{}", statement.sql_prefix, quote_identifier(owner, &DatabaseType::Postgres)) } pub async fn preview_transfer_ownership( state: &AppState, request: &TransferRequest, source_db_type: &DatabaseType, target_db_type: &DatabaseType, source_pool_key: &str, target_pool_key: &str, ) -> Result { if !request.create_table || !is_postgres_compat_transfer(source_db_type, target_db_type) { return Ok(TransferOwnershipPreview { missing_owners: Vec::new(), target_owner: String::new() }); } let has_prokind = postgres_transfer_catalog_capabilities(state, source_pool_key).await?.has_prokind; let relation_names = postgres_transfer_relation_names(request); let statements = get_postgres_ownership_statements_for_transfer( state, source_pool_key, &request.source_schema, &request.target_schema, &relation_names, has_prokind, ) .await?; let roles = distinct_postgres_ownership_roles(&statements); let existing_roles = get_existing_postgres_roles(state, target_pool_key, &roles).await?; let missing_owners = roles.into_iter().filter(|role| !existing_roles.contains(role)).collect::>(); let target_owner = if missing_owners.is_empty() { String::new() } else { get_postgres_current_user(state, target_pool_key).await? }; Ok(TransferOwnershipPreview { missing_owners, target_owner }) } fn postgres_transfer_grant_statements_sql( source_schema: &str, target_schema: &str, tables: &[String], has_prokind: bool, ) -> String { let table_list = tables.iter().map(|table| quote_string_literal(table)).collect::>().join(", "); let table_filter = if tables.is_empty() { "FALSE".to_string() } else { format!("c.relname IN ({table_list})") }; let (routine_kind, routine_filter) = postgres_transfer_routine_catalog_sql(has_prokind); format!( "WITH schema_grants AS ( \ SELECT format( \ 'GRANT %s ON SCHEMA %I TO %s%s', \ string_agg(a.privilege_type, ', ' ORDER BY a.privilege_type), \ {target_schema}, \ CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE quote_ident(grantee.rolname) END, \ CASE WHEN bool_or(a.is_grantable) THEN ' WITH GRANT OPTION' ELSE '' END \ ) AS stmt \ FROM ( \ SELECT n.nspname, (aclexplode(n.nspacl)).* \ FROM pg_catalog.pg_namespace n \ WHERE n.nspname = {source_schema} \ ) a \ LEFT JOIN pg_roles grantee ON grantee.oid = a.grantee \ GROUP BY a.grantee, grantee.rolname \ ), \ relation_grants AS ( \ SELECT format( \ 'GRANT %s ON %s %I.%I TO %s%s', \ string_agg(privilege_type, ', ' ORDER BY privilege_type), \ CASE WHEN relkind = 'S' THEN 'SEQUENCE' ELSE 'TABLE' END, \ {target_schema}, relname, \ CASE WHEN grantee = 0 THEN 'PUBLIC' ELSE quote_ident(rolname) END, \ CASE WHEN bool_or(is_grantable) THEN ' WITH GRANT OPTION' ELSE '' END \ ) AS stmt \ FROM ( \ SELECT a.relname, a.relkind, a.grantee, a.privilege_type, a.is_grantable, grantee.rolname \ FROM ( \ SELECT c.relname, c.relkind, (aclexplode(c.relacl)).* \ FROM pg_catalog.pg_class c \ JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ WHERE n.nspname = {source_schema} AND (c.relkind IN ('v','m') OR ({table_filter} AND c.relkind IN ('r','p','f','S'))) \ ) a \ LEFT JOIN pg_roles grantee ON grantee.oid = a.grantee \ ) rels \ GROUP BY relname, relkind, grantee, rolname \ ), \ routine_grants AS ( \ SELECT format( \ 'GRANT %s ON %s %I.%I(%s) TO %s%s', \ string_agg(privilege_type, ', ' ORDER BY privilege_type), \ routine_kind, \ {target_schema}, proname, identity_args, \ CASE WHEN grantee = 0 THEN 'PUBLIC' ELSE quote_ident(rolname) END, \ CASE WHEN bool_or(is_grantable) THEN ' WITH GRANT OPTION' ELSE '' END \ ) AS stmt \ FROM ( \ SELECT a.proname, a.routine_kind, a.identity_args, a.grantee, a.privilege_type, \ a.is_grantable, grantee.rolname \ FROM ( \ SELECT p.proname, {routine_kind} AS routine_kind, \ pg_get_function_identity_arguments(p.oid) AS identity_args, (aclexplode(p.proacl)).* \ FROM pg_catalog.pg_proc p \ JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace \ WHERE n.nspname = {source_schema} AND {routine_filter} \ ) a \ LEFT JOIN pg_roles grantee ON grantee.oid = a.grantee \ ) routines \ GROUP BY proname, routine_kind, identity_args, grantee, rolname \ ) \ SELECT stmt FROM ( \ SELECT stmt FROM schema_grants \ UNION ALL SELECT stmt FROM relation_grants \ UNION ALL SELECT stmt FROM routine_grants \ ) statements \ WHERE stmt IS NOT NULL", source_schema = quote_string_literal(source_schema), target_schema = quote_string_literal(target_schema), table_filter = table_filter, ) } async fn get_postgres_grant_statements_for_transfer( state: &AppState, pool_key: &str, source_schema: &str, target_schema: &str, tables: &[String], has_prokind: bool, ) -> Result, String> { let sql = postgres_transfer_grant_statements_sql(source_schema, target_schema, tables, has_prokind); Ok(result_rows_to_string_statements(execute_on_pool(state, pool_key, &sql).await?.rows)) } pub async fn is_cancelled(transfer_id: &str) -> bool { CANCELLED.read().await.contains(transfer_id) } pub async fn set_cancelled(transfer_id: &str) { CANCELLED.write().await.insert(transfer_id.to_string()); } pub async fn clear_cancelled(transfer_id: &str) { CANCELLED.write().await.remove(transfer_id); } /// Fetches full foreign key metadata for each of `tables`, one /// `list_foreign_keys_core` call per table. Always inserts an entry per input /// table (even when it has zero foreign keys), so callers can use /// `HashMap::get` to distinguish "checked, no FKs" from "not fetched" — the /// latter tells `transfer_table` it needs to fall back to a live query. async fn fetch_foreign_keys_for_tables( state: &AppState, connection_id: &str, database: &str, schema: &str, tables: &[String], ) -> Result>, String> { let mut result = HashMap::new(); for table in tables { let fks = crate::schema::list_foreign_keys_core(state, connection_id, database, schema, table).await?; result.insert(table.clone(), fks); } Ok(result) } /// Sort table names by foreign key dependency, also returning the full foreign /// key metadata fetched along the way (keyed by table name) so callers doing a /// data transfer don't have to re-query the same metadata per table later. /// /// When `parents_first` is true (data transfer / SQL export), referenced (parent) /// tables come before referencing (child) tables so inserts don't violate FK /// constraints. /// /// When `parents_first` is false (batch drop), referencing (child) tables come /// first so they are dropped before the tables they reference. /// /// Uses Kahn's algorithm for topological sort; tables involved in cycles keep /// their original relative order after all cycle-free tables. /// /// The returned map is empty when `tables.len() <= 1` (no fetch needed to sort) /// or when `connection_id` is a native Postgres connection (dependencies there /// come from a single batched `list_table_dependencies` query that doesn't /// build per-table `ForeignKeyInfo` — Postgres transfers don't consult this map). pub async fn sort_tables_by_fk_dependency_with_foreign_keys( state: &AppState, connection_id: &str, database: &str, schema: &str, tables: &[String], parents_first: bool, ) -> Result<(Vec, HashMap>), String> { if tables.len() <= 1 { return Ok((tables.to_vec(), HashMap::new())); } let db_type = state .configs .read() .await .get(connection_id) .map(|config| config.db_type) .ok_or_else(|| format!("Connection config not found: {connection_id}"))?; let postgres_pool = if db_type == DatabaseType::Postgres { let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?; { let pool_handle = state.pool_handle(&pool_key).await; native_postgres_dependency_pool(pool_handle.as_ref()) } } else { None }; let (dependencies, foreign_keys_by_table) = if let Some(pool) = postgres_pool { (db::postgres::list_table_dependencies(&pool, schema).await?, HashMap::new()) } else { let foreign_keys_by_table = fetch_foreign_keys_for_tables(state, connection_id, database, schema, tables).await?; let dependencies = foreign_keys_by_table .iter() .flat_map(|(table, fks)| fks.iter().map(move |fk| (table.clone(), fk.ref_table.clone()))) .collect::>(); (dependencies, foreign_keys_by_table) }; Ok((sort_table_names_by_dependencies(tables, &dependencies, parents_first), foreign_keys_by_table)) } /// Sort table names by foreign key dependency. See /// `sort_tables_by_fk_dependency_with_foreign_keys` for the full behavior — /// this is a thin wrapper that discards the fetched foreign key metadata, kept /// for callers that only need table order (batch drop, database export). pub async fn sort_tables_by_fk_dependency( state: &AppState, connection_id: &str, database: &str, schema: &str, tables: &[String], parents_first: bool, ) -> Result, String> { sort_tables_by_fk_dependency_with_foreign_keys(state, connection_id, database, schema, tables, parents_first) .await .map(|(sorted, _)| sorted) } fn native_postgres_dependency_pool(pool_kind: Option<&PoolKind>) -> Option { match pool_kind { Some(PoolKind::Postgres(pool)) => Some(pool.clone()), _ => None, } } pub(crate) fn sort_table_names_by_dependencies( tables: &[String], dependencies: &[(String, String)], parents_first: bool, ) -> Vec { let table_set: HashSet<&str> = tables.iter().map(|table| table.as_str()).collect(); // Build in-degree and dependents graph. // parents_first=true: edge ref_table → table (parent before child) // parents_first=false: edge table → ref_table (child before parent) let mut in_degree: HashMap<&str, usize> = HashMap::new(); let mut dependents: HashMap<&str, Vec<&str>> = HashMap::new(); for table in tables { in_degree.entry(table.as_str()).or_insert(0); } let mut seen_dependencies = HashSet::new(); for (table, ref_table) in dependencies { if !table_set.contains(table.as_str()) || !table_set.contains(ref_table.as_str()) { continue; } if !seen_dependencies.insert((table.as_str(), ref_table.as_str())) { continue; } if parents_first { // FK-bearing table depends on ref_table — parent comes first. *in_degree.entry(table.as_str()).or_insert(0) += 1; dependents.entry(ref_table.as_str()).or_default().push(table.as_str()); } else { // ref_table depends on FK-bearing table — child comes first. *in_degree.entry(ref_table.as_str()).or_insert(0) += 1; dependents.entry(table.as_str()).or_default().push(ref_table.as_str()); } } // Kahn's algorithm. let mut queue: std::collections::VecDeque<&str> = tables .iter() .map(String::as_str) .filter(|table| in_degree.get(table).copied().unwrap_or_default() == 0) .collect(); let mut sorted: Vec = Vec::new(); while let Some(table) = queue.pop_front() { sorted.push(table.to_string()); if let Some(deps) = dependents.get(table) { for &dependent in deps { let deg = in_degree.get_mut(dependent).unwrap(); *deg -= 1; if *deg == 0 { queue.push_back(dependent); } } } } // Append any tables left behind by cycles in their original order. if sorted.len() < tables.len() { let sorted_set: HashSet<&str> = sorted.iter().map(|s| s.as_str()).collect(); let mut remaining: Vec = Vec::new(); for table in tables { if !sorted_set.contains(table.as_str()) { remaining.push(table.clone()); } } sorted.extend(remaining); } sorted } #[allow(clippy::too_many_arguments)] async fn transfer_mongodb_table( state: &Arc, request: &TransferRequest, table: &str, table_index: usize, source_db_type: &DatabaseType, target_db_type: &DatabaseType, source_pool_key: &str, target_pool_key: &str, mut progress_callback: F, ) -> Result where F: FnMut(TransferProgress), { let total_tables = request.tables.len(); let ResolvedTransferTargetTable { name: target_table, preexisting: target_table_preexisting } = resolve_transfer_target_table_name( state, request, table, target_pool_key, target_db_type, request.source_catalog.as_deref(), request.target_catalog.as_deref(), ) .await; let batch_size = if request.batch_size == 0 { 1000 } else { request.batch_size }; let mut offset: u64 = 0; let mut total_transferred: u64 = 0; let mut total_rows = None; if request.mode == TransferMode::Upsert { log::warn!("[transfer] MongoDB upsert is not supported yet, falling back to append"); } if is_mongodb_transfer_type(target_db_type) && request.mode == TransferMode::Overwrite { overwrite_mongo_collection_for_transfer( state, &request.target_connection_id, &request.target_database, &target_table, ) .await .map_err(|e| format!("Failed to clear MongoDB collection '{target_table}': {e}"))?; } let mut sql_target_column_names: Vec = Vec::new(); let mut sql_target_column_types: Vec> = Vec::new(); let mut sql_target_prepared = false; loop { if is_cancelled(&request.transfer_id).await { return Err("Cancelled".to_string()); } let documents = if is_mongodb_transfer_type(source_db_type) { let result = if is_mongodb_transfer_type(target_db_type) { find_mongo_documents_extended_json( state, &request.source_connection_id, &request.source_database, table, offset, batch_size, ) .await? } else { find_mongo_documents_for_rows( state, &request.source_connection_id, &request.source_database, table, offset, batch_size, ) .await? }; total_rows = Some(result.total); mongo_documents_for_transfer(result, is_mongodb_transfer_type(target_db_type)) } else { let columns = get_columns_for_transfer( state, source_pool_key, &request.source_connection_id, &request.source_database, &request.source_schema, table, request.source_catalog.as_deref(), ) .await?; let col_names = columns.iter().map(|column| column.name.clone()).collect::>(); let primary_key_columns = transfer_key_columns(&columns, source_db_type); let sql = pagination_sql_with_order( &col_names, table, &request.source_schema, source_db_type, offset, batch_size, &primary_key_columns, request.source_catalog.as_deref(), ); let result = execute_on_pool(state, source_pool_key, &sql).await?; sql_rows_to_mongo_documents(&col_names, &result.rows) }; let row_count = documents.len(); if row_count == 0 { break; } if is_mongodb_transfer_type(target_db_type) { if is_mongodb_transfer_type(source_db_type) { insert_mongo_documents_extended_json_for_transfer( state, &request.target_connection_id, &request.target_database, &target_table, &documents, ) .await } else { insert_mongo_documents_for_transfer( state, &request.target_connection_id, &request.target_database, &target_table, &documents, ) .await } .map_err(|e| format!("Insert failed for MongoDB collection '{target_table}' at offset {offset}: {e}"))?; } else { if !sql_target_prepared { let mut sql_target_columns = mongo_columns_from_documents(&documents); if sql_target_columns.is_empty() { sql_target_columns.push(db::ColumnInfo { name: "document".to_string(), data_type: "json".to_string(), is_nullable: true, column_default: None, is_primary_key: false, extra: None, comment: None, numeric_precision: None, numeric_scale: None, character_maximum_length: None, enum_values: None, ..Default::default() }); } sql_target_column_names = sql_target_columns.iter().map(|column| column.name.clone()).collect(); sql_target_column_types = sql_target_columns.iter().map(|column| Some(column.data_type.clone())).collect(); if request.create_table { if !target_table_preexisting { let ddl = generate_create_table_ddl_with_column_quoting( &sql_target_columns, &target_table, &request.source_schema, &request.target_schema, target_db_type, source_db_type, None, request.target_catalog.as_deref(), request.quote_target_column_names, ); let target_table_created = transfer_create_table_created( execute_on_pool(state, target_pool_key, &ddl).await.map(|_| ()), &format!("Failed to create table from MongoDB collection '{table}'"), )?; if target_table_created { for stmt in generate_comment_ddl_with_column_quoting( &sql_target_columns, &target_table, &request.target_schema, target_db_type, None, request.quote_target_column_names, ) { if let Err(e) = execute_on_pool(state, target_pool_key, &stmt).await { log::warn!( "[transfer] failed to set MongoDB transfer column comment for {}: {}", target_table, e ); } } } } else { log::info!( "[transfer] target table {} already exists, skipping create-table DDL", target_table ); } } if request.mode == TransferMode::Overwrite { let full_table = qualified_table( &target_table, &request.target_schema, target_db_type, request.target_catalog.as_deref(), ); let truncate_sql = match target_db_type { DatabaseType::Sqlite | DatabaseType::CloudflareD1 | DatabaseType::DuckDb => { format!("DELETE FROM {full_table}") } _ => format!("TRUNCATE TABLE {full_table}"), }; execute_on_pool(state, target_pool_key, &truncate_sql) .await .map_err(|e| format!("Failed to truncate MongoDB transfer target table: {e}"))?; } sql_target_prepared = true; } let rows = if sql_target_column_names.len() == 1 && sql_target_column_names[0] == "document" { documents.iter().map(|document| vec![document.clone()]).collect::>() } else { mongo_documents_to_rows(&documents, &sql_target_column_names) }; let write_statements = generate_transfer_write_sql_batches_with_column_quoting( &TransferMode::Append, &sql_target_column_names, &sql_target_column_types, &rows, &target_table, &request.target_schema, target_db_type, &[], request.target_catalog.as_deref(), false, false, request.quote_target_column_names, )?; for (statement_index, batch_sql) in write_statements.iter().enumerate() { execute_on_pool(state, target_pool_key, batch_sql).await.map_err(|e| { format!( "Insert failed for MongoDB collection '{target_table}' at offset {offset}, chunk {} of {}: {e}", statement_index + 1, write_statements.len() ) })?; } } total_transferred += row_count as u64; offset += row_count as u64; progress_callback(TransferProgress { transfer_id: request.transfer_id.clone(), table: table.to_string(), table_index, total_tables, rows_transferred: total_transferred, total_rows, status: TransferStatus::Running, error: None, terminal: false, }); if row_count < batch_size { break; } } Ok(total_transferred) } #[derive(Default)] struct HiveServerTransferCursor { started: bool, session_id: Option, } fn transfer_cursor_sql( columns: &[String], table: &str, schema: &str, db_type: &DatabaseType, catalog: Option<&str>, ) -> String { let full_table = qualified_table(table, schema, db_type, catalog); let col_list = columns.iter().map(|column| quote_identifier(column, db_type)).collect::>().join(", "); format!("SELECT {col_list} FROM {full_table}") } async fn fetch_hive_server_transfer_batch( state: &AppState, pool_key: &str, request: &TransferRequest, sql: &str, batch_size: usize, cursor: &mut HiveServerTransferCursor, ) -> Result { let query_timeout_secs = if cursor.started { 0 } else { let configs = state.configs.read().await; configs.get(&request.source_connection_id).map(|config| config.query_timeout_secs).unwrap_or(0) }; let pool_handle = state.pool_handle(pool_key).await; let Some(PoolKind::Agent(client)) = pool_handle.as_ref() else { return Err("Impala transfer requires an Agent connection".to_string()); }; let client = client.clone(); let mut client = client.lock().await; let result = if cursor.started { let session_id = cursor.session_id.as_deref().ok_or("Impala transfer cursor ended before the next page was requested")?; client.fetch_table_read_page::(session_id, batch_size).await? } else { cursor.started = true; client .start_table_read::(AgentTableReadStartParams { sql: sql.to_string(), database: Some(request.source_database.clone()), schema: (!request.source_schema.trim().is_empty()).then(|| request.source_schema.clone()), page_size: batch_size, max_rows: AGENT_PROTOCOL_MAX_ROWS, fetch_size: Some(batch_size), timeout_secs: (query_timeout_secs > 0).then_some(query_timeout_secs), }) .await? }; if result.has_more { cursor.session_id = result.session_id.clone().or_else(|| cursor.session_id.clone()); if cursor.session_id.is_none() { return Err("Impala transfer cursor did not return a session id for additional rows".to_string()); } } else { cursor.session_id = None; } Ok(result) } async fn close_hive_server_transfer_cursor(state: &AppState, pool_key: &str, cursor: &mut HiveServerTransferCursor) { let Some(session_id) = cursor.session_id.take() else { return; }; let pool_handle = state.pool_handle(pool_key).await; let Some(PoolKind::Agent(client)) = pool_handle.as_ref() else { return; }; let client = client.clone(); let mut client = client.lock().await; if let Err(error) = client.close_table_read_session::(&session_id).await { log::warn!("[transfer] failed to close Impala transfer cursor: {error}"); } } /// Transfer a single table. Returns rows transferred. /// `progress_callback` is invoked for progress updates. #[allow(clippy::too_many_arguments)] async fn transfer_table_inner( state: &Arc, request: &TransferRequest, table: &str, table_index: usize, source_db_type: &DatabaseType, target_db_type: &DatabaseType, source_pool_key: &str, target_pool_key: &str, known_foreign_keys: &HashMap>, pending_fk_alters: &mut Vec<(String, String)>, mut progress_callback: F, ) -> Result where F: FnMut(TransferProgress), { if is_mongodb_transfer_type(source_db_type) || is_mongodb_transfer_type(target_db_type) { return transfer_mongodb_table( state, request, table, table_index, source_db_type, target_db_type, source_pool_key, target_pool_key, progress_callback, ) .await; } let total_tables = request.tables.len(); let pg_compat_transfer = is_postgres_compat_transfer(source_db_type, target_db_type); let ResolvedTransferTargetTable { name: target_table, preexisting: mut target_table_preexisting } = resolve_transfer_target_table_name( state, request, table, target_pool_key, target_db_type, request.source_catalog.as_deref(), request.target_catalog.as_deref(), ) .await; let preserves_target_table_name = target_table == table; // Get source columns (deduplicate by name) let columns = { let raw = get_columns_for_transfer( state, source_pool_key, &request.source_connection_id, &request.source_database, &request.source_schema, table, request.source_catalog.as_deref(), ) .await?; let mut seen = std::collections::HashSet::new(); raw.into_iter().filter(|c| seen.insert(c.name.clone())).collect::>() }; if columns.is_empty() { return Err(format!("No columns found for table {table}")); } let writable_columns = writable_transfer_columns(&columns, source_db_type, target_db_type); if writable_columns.is_empty() { return Err(format!("No writable columns found for table {table}")); } let col_names: Vec = writable_columns.iter().map(|c| c.name.clone()).collect(); let col_types: Vec> = writable_columns.iter().map(|c| Some(c.data_type.clone())).collect(); let primary_key_columns = transfer_key_columns(&writable_columns, source_db_type); if should_copy_data(&request.content) { log::info!("[transfer] {} has {} columns, counting rows...", table, columns.len()); } // Fetch source table comment. Keep this list-tables metadata chain on its // own task stack, just like the target-table lookup above. let table_comment = list_transfer_tables_isolated( state.clone(), request.source_connection_id.clone(), request.source_database.clone(), request.source_schema.clone(), request.source_catalog.clone(), *source_db_type, table.to_string(), 1, ) .await .unwrap_or_default() .into_iter() .next() .and_then(|table| table.comment); let source_indexes = if request.create_table && pg_compat_transfer && preserves_target_table_name && !target_table_preexisting { get_postgres_indexes_for_transfer( state, source_pool_key, &request.source_database, &request.source_schema, table, ) .await? } else { Vec::new() }; let source_foreign_keys = if request.create_table && pg_compat_transfer && preserves_target_table_name && !target_table_preexisting { get_postgres_foreign_keys_for_transfer( state, source_pool_key, &request.source_database, &request.source_schema, table, ) .await? } else { Vec::new() }; let total_rows = if should_copy_data(&request.content) { // Count source rows only for data-bearing transfers. let sql = count_sql(table, &request.source_schema, source_db_type, request.source_catalog.as_deref()); match execute_on_pool(state, source_pool_key, &sql).await { Ok(result) => result.rows.first().and_then(|r| r.first()).and_then(|v| match v { serde_json::Value::Number(n) => n.as_u64(), serde_json::Value::String(s) => s.parse::().ok(), _ => None, }), Err(e) => { log::warn!("[transfer] count failed for {}: {}", table, e); None } } } else { None }; log::info!("[transfer] {} total_rows={:?}", table, total_rows); // Create table on target if requested if request.create_table { if transfer_table_needs_inline_postgres_schema_ensure(source_db_type, target_db_type) && !request.target_schema.trim().is_empty() { let create_schema_sql = format!("CREATE SCHEMA IF NOT EXISTS {}", quote_identifier(&request.target_schema, target_db_type)); execute_on_pool(state, target_pool_key, &create_schema_sql) .await .map_err(|e| format!("Failed to ensure schema exists: {e}"))?; } if target_table_preexisting { log::info!("[transfer] target table {} already exists, skipping create-table DDL", target_table); } else { let owned_sequences = prepare_postgres_owned_sequences_for_transfer( state, request, table, &target_table, source_pool_key, target_pool_key, pg_compat_transfer, preserves_target_table_name, target_table_preexisting, ) .await?; let (source_driver_profile, target_driver_profile) = { let configs = state.configs.read().await; ( configs.get(&request.source_connection_id).and_then(|config| config.driver_profile.clone()), configs.get(&request.target_connection_id).and_then(|config| config.driver_profile.clone()), ) }; let can_reuse_source_ddl = can_reuse_source_table_ddl( source_db_type, target_db_type, source_driver_profile.as_deref(), target_driver_profile.as_deref(), preserves_target_table_name, ) && (request.quote_target_column_names || !matches!(target_db_type, DatabaseType::Gaussdb | DatabaseType::OpenGauss)); let mut reused_source_ddl = false; let ddl = if can_reuse_source_ddl { let (source_ddl, source_ddl_was_read) = if let Some(catalog) = resolve_external_transfer_catalog(request.source_catalog.as_deref(), source_db_type) { // Doris/StarRocks external catalog: read DDL directly via // SHOW CREATE TABLE catalog.database.table using the // existing source pool (bare MySQL — addresses any catalog). let pool = { let pool = state .pool_handle(source_pool_key) .await .ok_or_else(|| "Source pool not found".to_string())?; let PoolKind::Mysql(p, _) = &pool else { return Err("Source pool must be MySQL-family for catalog DDL".to_string()); }; p.clone() }; match db::doris::get_catalog_table_ddl(&pool, catalog, &request.source_database, table).await { Ok(ddl) => (ddl, true), Err(err) => { log::warn!("[transfer] catalog DDL read failed for {table} in catalog '{catalog}': {err}; falling back to generated DDL"); ( generate_create_table_ddl_with_column_quoting( &columns, &target_table, &request.source_schema, &request.target_schema, target_db_type, source_db_type, table_comment.as_deref(), request.target_catalog.as_deref(), request.quote_target_column_names, ), false, ) } } } else { match crate::schema::get_table_ddl_core( state, &request.source_connection_id, &request.source_database, &request.source_schema, table, None, ) .await { Ok(ddl) => (ddl, true), Err(_) => ( generate_create_table_ddl_with_column_quoting( &columns, &target_table, &request.source_schema, &request.target_schema, target_db_type, source_db_type, table_comment.as_deref(), request.target_catalog.as_deref(), request.quote_target_column_names, ), false, ), } }; if contains_oceanbase_mysql_table_options(&source_ddl) && !db::oceanbase_mysql::is_profile(target_db_type, target_driver_profile.as_deref()) { generate_create_table_ddl_with_column_quoting( &columns, &target_table, &request.source_schema, &request.target_schema, target_db_type, source_db_type, table_comment.as_deref(), request.target_catalog.as_deref(), request.quote_target_column_names, ) } else { reused_source_ddl = source_ddl_was_read; rewrite_transfer_source_table_ddl( &source_ddl, &request.source_schema, &request.target_schema, source_db_type, target_db_type, ) } } else { generate_create_table_ddl_with_column_quoting( &columns, &target_table, &request.source_schema, &request.target_schema, target_db_type, source_db_type, table_comment.as_deref(), request.target_catalog.as_deref(), request.quote_target_column_names, ) }; // MySQL-family targets: create the bare table first and add any foreign // keys via ALTER TABLE afterward, instead of relying on inline // `CREATE TABLE ... FOREIGN KEY` constraints. Inline FKs require every // referenced table to already exist, which the dependency sort can't // always guarantee (foreign key cycles have no valid creation order at // all) — mirrors the same defer-FK-creation approach already used for // Postgres transfers. let mut ddl = ddl; let mut deferred_fk_alters: Vec = Vec::new(); if supports_deferred_mysql_foreign_keys(target_db_type) { // Reuse the FK metadata `sort_tables_by_fk_dependency_with_foreign_keys` // already fetched for the whole batch when the caller provided it; // only fall back to a live query for callers that don't pre-fetch // (tests, or a native-Postgres source where the sort path takes a // different, cheaper route that doesn't build per-table FK lists). let foreign_keys = if let Some(fks) = known_foreign_keys.get(table) { Ok(fks.clone()) } else { crate::schema::list_foreign_keys_core( state, &request.source_connection_id, &request.source_database, &request.source_schema, table, ) .await }; match foreign_keys { Ok(foreign_keys) if !foreign_keys.is_empty() => { ddl = strip_inline_foreign_key_constraint_lines(&ddl); deferred_fk_alters = generate_mysql_foreign_key_alter_statements( &foreign_keys, request, &target_table, target_db_type, ); } Ok(_) => {} Err(e) => { log::warn!("[transfer] failed to inspect source foreign keys for {table}: {e}"); } } } log::info!("[transfer] creating target table: {}", ddl.chars().take(200).collect::()); let target_table_created = transfer_create_table_created( execute_transfer_create_table_ddl_on_pool( state, target_pool_key, &ddl, target_db_type, reused_source_ddl, ) .await, "Failed to create table", )?; if target_table_created { pending_fk_alters .extend(deferred_fk_alters.into_iter().map(|statement| (target_table.clone(), statement))); let comment_stmts = generate_comment_ddl_with_column_quoting( &columns, &target_table, &request.target_schema, target_db_type, table_comment.as_deref(), request.quote_target_column_names, ); for stmt in &comment_stmts { if let Err(e) = execute_on_pool(state, target_pool_key, stmt).await { log::warn!("[transfer] failed to set column comment for {}: {}", target_table, e); } } bind_postgres_owned_sequences_for_transfer( state, request, &target_table, target_pool_key, &owned_sequences, ) .await?; } else { // DDL may report the table already exists even when metadata // lookup missed it (case/schema differences or localized errors). target_table_preexisting = true; } } } let should_restore_postgres_table_schema = request.create_table && pg_compat_transfer && preserves_target_table_name && !target_table_preexisting; // Structure-only transfer: complete the table's post-create schema DDL, // then skip everything data-related. if !should_copy_data(&request.content) { if request.create_table && target_table_preexisting { let target_columns = get_columns_for_transfer( state, target_pool_key, &request.target_connection_id, &request.target_database, &request.target_schema, &target_table, request.target_catalog.as_deref(), ) .await .map_err(|error| { format!("Failed to inspect target table '{target_table}' columns before transfer: {error}") })?; validate_preexisting_target_columns( &target_columns, &col_names, target_db_type, request.quote_target_column_names, &target_table, )?; } if should_restore_postgres_table_schema { restore_postgres_table_schema_objects( state, target_pool_key, &target_table, &request.source_schema, &request.target_schema, &source_indexes, &source_foreign_keys, ) .await?; } return Ok(0); } let needs_target_columns = (request.create_table && target_table_preexisting) || (request.mode == TransferMode::Upsert && !matches!( target_db_type, DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala | DatabaseType::Argo )) || matches!(target_db_type, DatabaseType::Postgres | DatabaseType::Dameng); let target_columns = if needs_target_columns { get_columns_for_transfer( state, target_pool_key, &request.target_connection_id, &request.target_database, &request.target_schema, &target_table, request.target_catalog.as_deref(), ) .await .map_err(|error| format!("Failed to inspect target table '{target_table}' columns before transfer: {error}"))? } else { Vec::new() }; // The user asked DBX to sync structure (create_table), but the target // table already existed so the create-table DDL above was skipped (see // "skipping create-table DDL" above). If the untouched target structure // can't accept the planned insert, fail fast here instead of truncating // the target's existing data and then hitting an opaque driver error. if request.create_table && target_table_preexisting { validate_preexisting_target_columns( &target_columns, &col_names, target_db_type, request.quote_target_column_names, &target_table, )?; } // Truncate target if overwrite mode if request.mode == TransferMode::Overwrite { let full_table = qualified_table(&target_table, &request.target_schema, target_db_type, request.target_catalog.as_deref()); let truncate_sql = match target_db_type { DatabaseType::Sqlite | DatabaseType::CloudflareD1 | DatabaseType::DuckDb => { format!("DELETE FROM {full_table}") } _ => format!("TRUNCATE TABLE {full_table}"), }; execute_on_pool(state, target_pool_key, &truncate_sql).await.map_err(|e| format!("Failed to truncate: {e}"))?; } // Determine effective mode and PK columns for upsert let (effective_mode, pk_columns) = if request.mode == TransferMode::Upsert { if matches!( target_db_type, DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala | DatabaseType::Argo ) { log::warn!("[transfer] upsert not supported for {:?}, falling back to append", target_db_type); (TransferMode::Append, vec![]) } else { let pks: Vec = transfer_key_columns(&target_columns, target_db_type) .into_iter() .filter(|name| col_names.iter().any(|column_name| column_name.eq_ignore_ascii_case(name))) .collect(); if pks.is_empty() { log::warn!("[transfer] table {} has no primary key, falling back to append", table); (TransferMode::Append, vec![]) } else { (TransferMode::Upsert, pks) } } } else { (request.mode.clone(), vec![]) }; let writes_identity_insert_columns = matches!(target_db_type, DatabaseType::Dameng | DatabaseType::SqlServer) && selected_columns_include_identity_columns(&col_names, &target_columns); let overrides_postgres_system_values = matches!(target_db_type, DatabaseType::Postgres) && selected_columns_include_postgres_generated_always_identity_columns(&col_names, &target_columns); // Transfer data in batches let batch_size = if request.batch_size == 0 { 1000 } else { request.batch_size }; let mut offset: u64 = 0; let mut total_transferred: u64 = 0; // A single Agent cursor keeps Kyuubi/Impala rows in one query execution. // Re-running LIMIT/OFFSET pages is unstable for tables without a unique key. let use_hive_server_cursor = matches!(source_db_type, DatabaseType::Kyuubi | DatabaseType::Impala); let hive_server_transfer_sql = use_hive_server_cursor.then(|| { transfer_cursor_sql( &col_names, table, &request.source_schema, source_db_type, request.source_catalog.as_deref(), ) }); let mut hive_server_cursor = HiveServerTransferCursor::default(); let transfer_result: Result<(), String> = async { loop { if is_cancelled(&request.transfer_id).await { return Err("Cancelled".to_string()); } let (result, mysql_spatial_markers) = if let Some(sql) = hive_server_transfer_sql.as_deref() { ( fetch_hive_server_transfer_batch( state, source_pool_key, request, sql, batch_size, &mut hive_server_cursor, ) .await?, false, ) } else { let sql = pagination_sql_with_order( &col_names, table, &request.source_schema, source_db_type, offset, batch_size, &primary_key_columns, request.source_catalog.as_deref(), ); let (sql, mysql_spatial_markers) = mysql_spatial_transfer_select_sql(sql, &col_names, &col_types, source_db_type, target_db_type); (execute_on_pool(state, source_pool_key, &sql).await?, mysql_spatial_markers) }; let has_more = result.has_more; let row_count = result.rows.len(); if row_count == 0 { break; } let write_statements = generate_transfer_write_sql_batches_with_column_quoting( &effective_mode, &col_names, &col_types, &result.rows, &target_table, &request.target_schema, target_db_type, &pk_columns, request.target_catalog.as_deref(), overrides_postgres_system_values, mysql_spatial_markers, request.quote_target_column_names, )?; for (statement_index, batch_sql) in write_statements.iter().enumerate() { execute_transfer_write_statement( state, target_pool_key, batch_sql, target_db_type, &target_table, &request.target_schema, writes_identity_insert_columns, ) .await .map_err(|e| { let absolute_row = parse_mysql_row_error(&e).map(|row| offset + row); match absolute_row { Some(row) => format!( "Insert failed for table '{target_table}' at row {row} (chunk {} of {}): {e}", statement_index + 1, write_statements.len() ), None => format!( "Insert failed for table '{target_table}' at offset {offset}, chunk {} of {}: {e}", statement_index + 1, write_statements.len() ), } })?; } total_transferred += row_count as u64; log::info!("[transfer] {} batch +{} rows (total {})", table, row_count, total_transferred); offset += row_count as u64; progress_callback(TransferProgress { transfer_id: request.transfer_id.clone(), table: table.to_string(), table_index, total_tables, rows_transferred: total_transferred, total_rows, status: TransferStatus::Running, error: None, terminal: false, }); if (use_hive_server_cursor && !has_more) || (!use_hive_server_cursor && row_count < batch_size) { break; } } Ok(()) } .await; close_hive_server_transfer_cursor(state, source_pool_key, &mut hive_server_cursor).await; transfer_result?; if pg_compat_transfer { for statement in generate_postgres_sequence_sync_sql(&columns, &target_table, &request.target_schema) { execute_on_pool(state, target_pool_key, &statement) .await .map_err(|e| format!("Failed to sync PostgreSQL sequence for {target_table}: {e}"))?; } } if should_restore_postgres_table_schema { restore_postgres_table_schema_objects( state, target_pool_key, &target_table, &request.source_schema, &request.target_schema, &source_indexes, &source_foreign_keys, ) .await?; } Ok(total_transferred) } /// Transfer one table on its own Tokio task so the large transfer future and /// nested driver metadata futures do not share a single worker stack. #[allow(clippy::too_many_arguments)] pub async fn transfer_table( state: &Arc, request: &TransferRequest, table: &str, table_index: usize, source_db_type: &DatabaseType, target_db_type: &DatabaseType, source_pool_key: &str, target_pool_key: &str, known_foreign_keys: &HashMap>, pending_fk_alters: &mut Vec<(String, String)>, mut progress_callback: F, ) -> Result where F: FnMut(TransferProgress), { let state = state.clone(); let request = request.clone(); let table = table.to_string(); let source_db_type = *source_db_type; let target_db_type = *target_db_type; let source_pool_key = source_pool_key.to_string(); let target_pool_key = target_pool_key.to_string(); let known_foreign_keys = known_foreign_keys .get(&table) .map(|foreign_keys| HashMap::from([(table.clone(), foreign_keys.clone())])) .unwrap_or_default(); let (progress_tx, mut progress_rx) = tokio::sync::mpsc::channel(TRANSFER_PROGRESS_CHANNEL_CAPACITY); let mut task = tokio::spawn(async move { let mut task_pending_fk_alters = Vec::new(); let result = transfer_table_inner( &state, &request, &table, table_index, &source_db_type, &target_db_type, &source_pool_key, &target_pool_key, &known_foreign_keys, &mut task_pending_fk_alters, move |progress| { try_send_transfer_progress(&progress_tx, progress); }, ) .await; (result, task_pending_fk_alters) }); let _abort_on_drop = AbortTransferTaskOnDrop(task.abort_handle()); loop { tokio::select! { biased; Some(progress) = progress_rx.recv() => progress_callback(progress), result = &mut task => { let (result, task_pending_fk_alters) = result.map_err(|error| format!("Transfer table task failed: {error}"))?; pending_fk_alters.extend(task_pending_fk_alters); return result; } } } } pub async fn transfer_postgres_schema_dependencies( state: &AppState, request: &TransferRequest, source_pool_key: &str, target_pool_key: &str, mut progress_callback: F, ) -> Result<(), String> where F: FnMut(TransferProgress), { let source_db_type = get_db_type(state, &request.source_connection_id).await?; let target_db_type = get_db_type(state, &request.target_connection_id).await?; if !request.create_table || !is_postgres_compat_transfer(&source_db_type, &target_db_type) { return Ok(()); } if !request.target_schema.trim().is_empty() { let schema_exists = execute_on_pool(state, target_pool_key, &postgres_schema_exists_sql(&request.target_schema)) .await .map_err(|e| format!("Failed to check PostgreSQL target schema: {e}"))?; if !query_result_has_rows(&schema_exists) { // CREATE SCHEMA requires database-level CREATE privilege even with // IF NOT EXISTS, so only issue it after confirming the schema is absent. let create_schema_sql = format!("CREATE SCHEMA {}", quote_identifier(&request.target_schema, &DatabaseType::Postgres)); execute_on_pool(state, target_pool_key, &create_schema_sql) .await .map_err(|e| format!("Failed to create PostgreSQL target schema: {e}"))?; } } let extensions = get_postgres_extension_sources_for_transfer(state, source_pool_key, &request.source_schema).await?; let enum_types = get_postgres_enum_sources_for_transfer(state, source_pool_key, &request.source_schema).await?; let domains = get_postgres_domain_sources_for_transfer(state, source_pool_key, &request.source_schema).await?; let selected_sequence_names = selected_postgres_sequence_names(request); let selected_sequences = get_postgres_selected_sequences_for_transfer( state, source_pool_key, &request.source_schema, &selected_sequence_names, ) .await?; let existing_sequence_names = get_existing_postgres_sequence_names_for_transfer( state, target_pool_key, &request.target_schema, &selected_sequence_names, ) .await?; let selected_sequences = selected_sequences .into_iter() .filter(|sequence| !existing_sequence_names.contains(&sequence.name)) .collect::>(); let total_steps = extensions.len() + enum_types.len() + domains.len() + selected_sequences.len(); let table_index = 0; let mut completed_steps = 0_u64; for extension in extensions { if is_cancelled(&request.transfer_id).await { return Err("Cancelled".to_string()); } completed_steps += 1; progress_callback(TransferProgress { transfer_id: request.transfer_id.clone(), table: format!("extension: {}", extension.extension_name), table_index, total_tables: request.tables.len(), rows_transferred: completed_steps, total_rows: Some(total_steps as u64), status: TransferStatus::Running, error: None, terminal: false, }); execute_on_pool(state, target_pool_key, &generate_postgres_extension_ddl(&extension, &request.target_schema)) .await .map_err(|e| format!("Failed to create PostgreSQL extension {}: {e}", extension.extension_name))?; } for enum_type in enum_types { if is_cancelled(&request.transfer_id).await { return Err("Cancelled".to_string()); } completed_steps += 1; progress_callback(TransferProgress { transfer_id: request.transfer_id.clone(), table: format!("enum: {}", enum_type.type_name), table_index, total_tables: request.tables.len(), rows_transferred: completed_steps, total_rows: Some(total_steps as u64), status: TransferStatus::Running, error: None, terminal: false, }); execute_on_pool(state, target_pool_key, &generate_postgres_enum_ddl(&enum_type, &request.target_schema)) .await .map_err(|e| format!("Failed to create PostgreSQL enum {}: {e}", enum_type.type_name))?; } for domain in domains { if is_cancelled(&request.transfer_id).await { return Err("Cancelled".to_string()); } completed_steps += 1; progress_callback(TransferProgress { transfer_id: request.transfer_id.clone(), table: format!("domain: {}", domain.domain_name), table_index, total_tables: request.tables.len(), rows_transferred: completed_steps, total_rows: Some(total_steps as u64), status: TransferStatus::Running, error: None, terminal: false, }); execute_on_pool(state, target_pool_key, &generate_postgres_domain_ddl(&domain, &request.target_schema)) .await .map_err(|e| format!("Failed to create PostgreSQL domain {}: {e}", domain.domain_name))?; } for sequence in selected_sequences { if is_cancelled(&request.transfer_id).await { return Err("Cancelled".to_string()); } completed_steps += 1; progress_callback(TransferProgress { transfer_id: request.transfer_id.clone(), table: format!("sequence: {}", sequence.name), table_index, total_tables: request.tables.len(), rows_transferred: completed_steps, total_rows: Some(total_steps as u64), status: TransferStatus::Running, error: None, terminal: false, }); execute_on_pool( state, target_pool_key, &generate_postgres_transfer_sequence_create_ddl(&sequence, &request.target_schema), ) .await .map_err(|e| format!("Failed to create PostgreSQL sequence {}: {e}", sequence.name))?; if let Some(setval_sql) = generate_postgres_transfer_sequence_setval_sql(&sequence, &request.target_schema) { execute_on_pool(state, target_pool_key, &setval_sql) .await .map_err(|e| format!("Failed to restore PostgreSQL sequence {} value: {e}", sequence.name))?; } } Ok(()) } pub async fn transfer_postgres_schema_objects( state: &AppState, request: &TransferRequest, source_pool_key: &str, target_pool_key: &str, mut progress_callback: F, ) -> Result where F: FnMut(TransferProgress), { let source_db_type = get_db_type(state, &request.source_connection_id).await?; let target_db_type = get_db_type(state, &request.target_connection_id).await?; if !request.create_table || !is_postgres_compat_transfer(&source_db_type, &target_db_type) { return Ok(TransferObjectOutcome::default()); } let catalog_capabilities = postgres_transfer_catalog_capabilities(state, source_pool_key).await?; let has_prokind = catalog_capabilities.has_prokind; let mut outcome = TransferObjectOutcome::default(); let object_sources = filter_object_sources_by_selection( get_postgres_schema_object_sources_for_transfer(state, source_pool_key, &request.source_schema, has_prokind) .await?, &request.objects, ); let materialized_views = get_postgres_materialized_view_sources_for_transfer(state, source_pool_key, &request.source_schema) .await? .into_iter() .filter(|view| { object_kind_selected_or_defaulted(&request.objects, &TransferObjectKind::MaterializedView) && selected_object_names(&request.objects, &TransferObjectKind::MaterializedView) .contains(&view.view_name) }) .collect::>(); let trigger_sources = get_postgres_trigger_sources_for_transfer(state, source_pool_key, &request.source_schema, &request.tables) .await? .into_iter() .filter(|trigger| { object_kind_selected_or_defaulted(&request.objects, &TransferObjectKind::Trigger) && selected_object_names(&request.objects, &TransferObjectKind::Trigger) .contains(&trigger.trigger_name) }) .collect::>(); let policy_statements = get_postgres_policy_statements_for_transfer( state, source_pool_key, &request.source_schema, &request.target_schema, &request.tables, catalog_capabilities.has_pg_policy, catalog_capabilities.supports_policy_permissiveness, ) .await?; let relation_names = postgres_transfer_relation_names(request); let ownership_statements = if matches!(request.ownership_policy, TransferOwnershipPolicy::Skip) { Vec::new() } else { get_postgres_ownership_statements_for_transfer( state, source_pool_key, &request.source_schema, &request.target_schema, &relation_names, has_prokind, ) .await? }; let ownership_existing_roles = if matches!(request.ownership_policy, TransferOwnershipPolicy::ReassignMissing) { let roles = distinct_postgres_ownership_roles(&ownership_statements); get_existing_postgres_roles(state, target_pool_key, &roles).await? } else { HashSet::new() }; let ownership_target_user = if matches!(request.ownership_policy, TransferOwnershipPolicy::ReassignMissing) && !ownership_statements.is_empty() { Some(get_postgres_current_user(state, target_pool_key).await?) } else { None }; let grant_statements = get_postgres_grant_statements_for_transfer( state, source_pool_key, &request.source_schema, &request.target_schema, &relation_names, has_prokind, ) .await?; let materialized_view_step_count = materialized_views .iter() .map(|view| generate_postgres_materialized_view_ddls(view, &request.target_schema).len()) .sum::(); let trigger_step_count = trigger_sources.len() * 2; let total_steps = object_sources.len() + materialized_view_step_count + trigger_step_count + policy_statements.len() + ownership_statements.len() + grant_statements.len(); let table_index = request.tables.len(); let mut completed_steps = 0_u64; for object in object_sources { if is_cancelled(&request.transfer_id).await { return Err("Cancelled".to_string()); } let Some(object_kind) = transfer_kind_from_object_source_kind(&object.object_type) else { continue; }; let exists_sql = target_object_exists_sql(&DatabaseType::Postgres, &request.target_schema, &object.name, &object_kind)?; let exists = !execute_on_pool(state, target_pool_key, &exists_sql).await?.rows.is_empty(); if exists { outcome.skipped.push(format!("{object_kind:?}:{}", object.name)); continue; } completed_steps += 1; progress_callback(TransferProgress { transfer_id: request.transfer_id.clone(), table: format!("schema object: {}", object.name), table_index, total_tables: request.tables.len(), rows_transferred: completed_steps, total_rows: Some(total_steps as u64), status: TransferStatus::Running, error: None, terminal: false, }); let rewritten_source = match object.object_type { db::ObjectSourceKind::View | db::ObjectSourceKind::MaterializedView => object.source.clone(), db::ObjectSourceKind::Procedure | db::ObjectSourceKind::Function => { rewrite_postgres_routine_schema(&object.source, &request.source_schema, &request.target_schema) .unwrap_or_else(|| object.source.clone()) } db::ObjectSourceKind::Sequence | db::ObjectSourceKind::Synonym | db::ObjectSourceKind::Job | db::ObjectSourceKind::Package | db::ObjectSourceKind::PackageBody => object.source.clone(), db::ObjectSourceKind::Trigger | db::ObjectSourceKind::Event | db::ObjectSourceKind::Type | db::ObjectSourceKind::TypeBody => object.source.clone(), }; let statements = build_executable_object_source_statements(EditableObjectSourceSqlInput { database_type: DatabaseType::Postgres, object_type: object.object_type.clone(), schema: Some(request.target_schema.clone()), name: object.name.clone(), source: rewritten_source, })?; for statement in statements { execute_on_pool(state, target_pool_key, &statement) .await .map_err(|e| format!("Failed to create PostgreSQL {:?} {}: {e}", object.object_type, object.name))?; } outcome.transferred.push(format!("{object_kind:?}:{}", object.name)); } for view in materialized_views { let exists_sql = target_object_exists_sql( &DatabaseType::Postgres, &request.target_schema, &view.view_name, &TransferObjectKind::MaterializedView, )?; let exists = !execute_on_pool(state, target_pool_key, &exists_sql).await?.rows.is_empty(); if exists { outcome.skipped.push(format!("MaterializedView:{}", view.view_name)); continue; } for statement in generate_postgres_materialized_view_ddls(&view, &request.target_schema) { if is_cancelled(&request.transfer_id).await { return Err("Cancelled".to_string()); } completed_steps += 1; progress_callback(TransferProgress { transfer_id: request.transfer_id.clone(), table: format!("materialized view: {}", view.view_name), table_index, total_tables: request.tables.len(), rows_transferred: completed_steps, total_rows: Some(total_steps as u64), status: TransferStatus::Running, error: None, terminal: false, }); execute_on_pool(state, target_pool_key, &statement) .await .map_err(|e| format!("Failed to create PostgreSQL materialized view {}: {e}", view.view_name))?; } outcome.transferred.push(format!("MaterializedView:{}", view.view_name)); } for trigger in trigger_sources { if is_cancelled(&request.transfer_id).await { return Err("Cancelled".to_string()); } let exists_sql = target_object_exists_sql( &DatabaseType::Postgres, &request.target_schema, &trigger.trigger_name, &TransferObjectKind::Trigger, )?; let exists = !execute_on_pool(state, target_pool_key, &exists_sql).await?.rows.is_empty(); if exists { outcome.skipped.push(format!("Trigger:{}", trigger.trigger_name)); continue; } completed_steps += 1; progress_callback(TransferProgress { transfer_id: request.transfer_id.clone(), table: format!("trigger: {}", trigger.trigger_name), table_index, total_tables: request.tables.len(), rows_transferred: completed_steps, total_rows: Some(total_steps as u64), status: TransferStatus::Running, error: None, terminal: false, }); let full_table = qualified_table(&trigger.table_name, &request.target_schema, &DatabaseType::Postgres, None); let drop_sql = format!( "DROP TRIGGER IF EXISTS {} ON {full_table}", quote_identifier(&trigger.trigger_name, &DatabaseType::Postgres) ); execute_on_pool(state, target_pool_key, &drop_sql) .await .map_err(|e| format!("Failed to drop PostgreSQL trigger {}: {e}", trigger.trigger_name))?; completed_steps += 1; progress_callback(TransferProgress { transfer_id: request.transfer_id.clone(), table: format!("trigger: {}", trigger.trigger_name), table_index, total_tables: request.tables.len(), rows_transferred: completed_steps, total_rows: Some(total_steps as u64), status: TransferStatus::Running, error: None, terminal: false, }); let create_sql = rewrite_postgres_trigger_table_schema( &ensure_sql_statement_terminated(&trigger.source), &request.source_schema, &trigger.table_name, &request.target_schema, ); execute_on_pool(state, target_pool_key, &create_sql) .await .map_err(|e| format!("Failed to create PostgreSQL trigger {}: {e}", trigger.trigger_name))?; outcome.transferred.push(format!("Trigger:{}", trigger.trigger_name)); } for statement in policy_statements { if is_cancelled(&request.transfer_id).await { return Err("Cancelled".to_string()); } completed_steps += 1; progress_callback(TransferProgress { transfer_id: request.transfer_id.clone(), table: "row security policies".to_string(), table_index, total_tables: request.tables.len(), rows_transferred: completed_steps, total_rows: Some(total_steps as u64), status: TransferStatus::Running, error: None, terminal: false, }); execute_on_pool(state, target_pool_key, &statement) .await .map_err(|e| format!("Failed to apply PostgreSQL row security statement: {e}"))?; } for statement in ownership_statements { if is_cancelled(&request.transfer_id).await { return Err("Cancelled".to_string()); } completed_steps += 1; progress_callback(TransferProgress { transfer_id: request.transfer_id.clone(), table: "ownership".to_string(), table_index, total_tables: request.tables.len(), rows_transferred: completed_steps, total_rows: Some(total_steps as u64), status: TransferStatus::Running, error: None, terminal: false, }); let ownership_owner = if matches!(request.ownership_policy, TransferOwnershipPolicy::ReassignMissing) && !ownership_existing_roles.contains(&statement.owner) { ownership_target_user .as_deref() .ok_or_else(|| "Failed to read target PostgreSQL current user".to_string())? } else { &statement.owner }; let ownership_sql = build_postgres_ownership_statement(&statement, ownership_owner); execute_on_pool(state, target_pool_key, &ownership_sql) .await .map_err(|e| format!("Failed to apply PostgreSQL ownership statement: {e}"))?; } for statement in grant_statements { if is_cancelled(&request.transfer_id).await { return Err("Cancelled".to_string()); } completed_steps += 1; progress_callback(TransferProgress { transfer_id: request.transfer_id.clone(), table: "grants".to_string(), table_index, total_tables: request.tables.len(), rows_transferred: completed_steps, total_rows: Some(total_steps as u64), status: TransferStatus::Running, error: None, terminal: false, }); execute_on_pool(state, target_pool_key, &statement) .await .map_err(|e| format!("Failed to apply PostgreSQL grant statement: {e}"))?; } Ok(outcome) } #[cfg(test)] mod tests { use super::*; fn jdbc_transfer_config(connection_string: &str, driver_class: &str, profile: &str) -> ConnectionConfig { ConnectionConfig { id: "test-jdbc".to_string(), name: "Test JDBC".to_string(), note: String::new(), db_type: DatabaseType::Jdbc, driver_profile: if profile.is_empty() { None } else { Some(profile.to_string()) }, driver_label: None, url_params: None, agent_java_options: Vec::new(), host: String::new(), port: 0, username: String::new(), password: String::new(), database: None, default_schema: None, visible_databases: None, visible_database_patterns: None, visible_schemas: None, show_system_schemas: false, attached_databases: Vec::new(), init_script: None, color: None, transport_layers: Vec::new(), connect_timeout_secs: 5, query_timeout_secs: 30, idle_timeout_secs: 60, keepalive_interval_secs: 0, ssl: false, ca_cert_path: String::new(), client_cert_path: String::new(), client_key_path: String::new(), sysdba: false, oracle_connection_type: None, connection_string: if connection_string.is_empty() { None } else { Some(connection_string.to_string()) }, redis_connection_mode: None, redis_sentinel_master: String::new(), redis_sentinel_nodes: String::new(), redis_sentinel_username: String::new(), redis_sentinel_password: String::new(), redis_sentinel_tls: false, redis_cluster_nodes: String::new(), redis_key_separator: crate::models::connection::default_redis_key_separator(), redis_scan_page_size: None, redis_database_aliases: Default::default(), redis_key_templates: Vec::new(), etcd_endpoints: String::new(), gbase_server: String::new(), informix_server: String::new(), external_config: None, jdbc_driver_class: if driver_class.is_empty() { None } else { Some(driver_class.to_string()) }, jdbc_driver_paths: Vec::new(), one_time: false, save_password: true, read_only: false, is_production: false, production_databases: vec![], database_info: None, docs_notes_path: None, } } #[test] fn gbase_8s_jdbc_stays_generic_for_transfer() { assert_eq!( effective_transfer_database_type(&jdbc_transfer_config( "jdbc:gbasedbt-sqli://localhost:9088/dbx_test:INFORMIXSERVER=ol_gbasedbt", "com.gbasedbt.jdbc.Driver", "" )), DatabaseType::Jdbc ); assert_eq!( effective_transfer_database_type(&jdbc_transfer_config( "jdbc:gbase://localhost:5258/dbx_test", "cn.gbase.Driver", "gbase8s" )), DatabaseType::Jdbc ); assert_eq!( effective_transfer_database_type(&jdbc_transfer_config( "jdbc:gbase://localhost:5258/dbx_test", "com.gbase.jdbc.Driver", "" )), DatabaseType::Gbase ); } use serde_json::json; fn test_column(name: &str, data_type: &str) -> db::ColumnInfo { db::ColumnInfo { name: name.to_string(), data_type: data_type.to_string(), is_nullable: true, column_default: None, is_primary_key: false, extra: None, comment: None, numeric_precision: None, numeric_scale: None, character_maximum_length: None, enum_values: None, ..Default::default() } } fn test_table(name: &str) -> db::TableInfo { db::TableInfo { name: name.to_string(), table_type: "TABLE".to_string(), comment: None, parent_schema: None, parent_name: None, } } fn test_query_result(rows: Vec>) -> db::QueryResult { db::QueryResult { columns: Vec::new(), column_types: Vec::new(), column_sortables: Vec::new(), spatial_columns: vec![], spatial_values: vec![], rows, affected_rows: 0, execution_time_ms: 0, truncated: false, session_id: None, has_more: false, elasticsearch_raw_body: None, messages: Vec::new(), } } #[test] fn table_dependency_sort_places_parents_before_children() { let tables = vec!["audit".to_string(), "users".to_string(), "orders".to_string()]; let dependencies = vec![("audit".to_string(), "orders".to_string()), ("orders".to_string(), "users".to_string())]; assert_eq!( sort_table_names_by_dependencies(&tables, &dependencies, true), vec!["users".to_string(), "orders".to_string(), "audit".to_string()] ); assert_eq!( sort_table_names_by_dependencies(&tables, &dependencies, false), vec!["audit".to_string(), "orders".to_string(), "users".to_string()] ); } #[test] fn table_dependency_sort_ignores_duplicates_and_out_of_scope_tables() { let tables = vec!["orders".to_string(), "users".to_string(), "logs".to_string()]; let dependencies = vec![ ("orders".to_string(), "users".to_string()), ("orders".to_string(), "users".to_string()), ("logs".to_string(), "external_users".to_string()), ]; assert_eq!( sort_table_names_by_dependencies(&tables, &dependencies, true), vec!["users".to_string(), "logs".to_string(), "orders".to_string()] ); } #[test] fn postgres_dependency_batch_query_keeps_agent_fallback() { let agent = PoolKind::agent(crate::db::agent_driver::AgentDriverClient::test_stub()); assert!(native_postgres_dependency_pool(Some(&agent)).is_none()); } async fn test_app_state() -> (AppState, std::path::PathBuf) { let dir = std::env::temp_dir().join(format!("dbx-transfer-test-{}", uuid::Uuid::new_v4())); std::fs::create_dir_all(&dir).unwrap(); let storage = crate::storage::Storage::open(&dir.join("storage.db")).await.unwrap(); (AppState::new(storage), dir) } #[tokio::test] async fn postgres_transfer_metadata_routes_agent_pools() { let (state, dir) = test_app_state().await; state .update_connection_pools(|connections| { connections.insert( "source:source_db".to_string(), PoolKind::agent(crate::db::agent_driver::AgentDriverClient::test_stub()), ); }) .await; let index_error = get_postgres_indexes_for_transfer(&state, "source:source_db", "source_db", "source_schema", "items") .await .unwrap_err(); let foreign_key_error = get_postgres_foreign_keys_for_transfer(&state, "source:source_db", "source_db", "source_schema", "items") .await .unwrap_err(); assert!(!index_error.contains("PostgreSQL pool not found"), "index error: {index_error}"); assert!(!foreign_key_error.contains("PostgreSQL pool not found"), "foreign key error: {foreign_key_error}"); let _ = std::fs::remove_dir_all(dir); } #[test] fn postgres_schema_exists_query_escapes_schema_name() { assert_eq!( postgres_schema_exists_sql("team's data"), "SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = 'team''s data' LIMIT 1" ); } #[test] fn postgres_schema_exists_depends_on_returned_rows() { assert!(!query_result_has_rows(&test_query_result(Vec::new()))); assert!(query_result_has_rows(&test_query_result(vec![vec![serde_json::json!(1)]]))); } #[test] fn transfer_content_defaults_to_structure_and_data() { let request: TransferRequest = serde_json::from_value(serde_json::json!({ "transferId": "t1", "sourceConnectionId": "s", "sourceDatabase": "db", "sourceSchema": "public", "targetConnectionId": "t", "targetDatabase": "db", "targetSchema": "public", "tables": ["a"], "createTable": true, "mode": "append", "targetTableNameCase": "preserve", "batchSize": 1000 })) .unwrap(); assert_eq!(request.content, TransferContent::StructureAndData); assert!(request.objects.is_empty()); assert!(request.quote_target_column_names); } #[test] fn transfer_request_serializes_new_fields_camel_case() { let request = TransferRequest { transfer_id: "t1".to_string(), source_connection_id: "s".to_string(), source_database: "db".to_string(), source_schema: "public".to_string(), source_catalog: None, target_connection_id: "t".to_string(), target_database: "db".to_string(), target_schema: "public".to_string(), target_catalog: None, tables: vec!["a".to_string()], create_table: true, content: TransferContent::StructureOnly, objects: vec![TransferObjectSelection { object_type: TransferObjectKind::View, names: vec!["v1".to_string()], }], mode: TransferMode::Append, target_table_name_case: TransferTableNameCase::Preserve, quote_target_column_names: true, ownership_policy: TransferOwnershipPolicy::Preserve, batch_size: 1000, }; let json = serde_json::to_value(&request).unwrap(); assert_eq!(json["content"], "structureOnly"); assert_eq!(json["objects"][0]["objectType"], "VIEW"); assert_eq!(json["objects"][0]["names"][0], "v1"); assert_eq!(json["quoteTargetColumnNames"], true); } mod transfer_family_tests { use super::*; #[test] fn same_family_matrix() { // postgres family assert!(is_same_transfer_family(&DatabaseType::Postgres, &DatabaseType::Kingbase)); assert!(is_same_transfer_family(&DatabaseType::Gaussdb, &DatabaseType::OpenGauss)); // oracle family assert!(is_same_transfer_family(&DatabaseType::Oracle, &DatabaseType::Dameng)); assert!(is_same_transfer_family(&DatabaseType::OceanbaseOracle, &DatabaseType::Dameng)); // mysql assert!(is_same_transfer_family(&DatabaseType::Mysql, &DatabaseType::Mysql)); // sqlserver assert!(is_same_transfer_family(&DatabaseType::SqlServer, &DatabaseType::SqlServer)); // cross family assert!(!is_same_transfer_family(&DatabaseType::Mysql, &DatabaseType::Postgres)); assert!(!is_same_transfer_family(&DatabaseType::Mysql, &DatabaseType::SqlServer)); } #[test] fn object_kinds_per_family() { let mysql = transfer_object_kinds(&DatabaseType::Mysql); assert!(mysql.contains(&TransferObjectKind::Event)); assert!(!mysql.contains(&TransferObjectKind::Sequence)); let pg = transfer_object_kinds(&DatabaseType::Postgres); assert!(pg.contains(&TransferObjectKind::Sequence)); assert!(!pg.contains(&TransferObjectKind::Event)); let dm = transfer_object_kinds(&DatabaseType::Dameng); assert!(dm.contains(&TransferObjectKind::Trigger)); assert!(dm.contains(&TransferObjectKind::Sequence)); let sqlserver = transfer_object_kinds(&DatabaseType::SqlServer); assert!(sqlserver.contains(&TransferObjectKind::View)); assert!(sqlserver.contains(&TransferObjectKind::Procedure)); assert!(sqlserver.contains(&TransferObjectKind::Function)); assert!(sqlserver.contains(&TransferObjectKind::Trigger)); assert!(sqlserver.contains(&TransferObjectKind::Sequence)); assert!(!sqlserver.contains(&TransferObjectKind::Event)); assert!(!sqlserver.contains(&TransferObjectKind::MaterializedView)); assert!(transfer_object_kinds(&DatabaseType::Sqlite).is_empty()); } } mod transfer_validation_tests { use super::*; #[test] fn validates_content_and_object_rules() { let base = TransferRequest { transfer_id: "t".into(), source_connection_id: "s".into(), source_database: "db".into(), source_schema: "public".into(), source_catalog: None, target_connection_id: "t".into(), target_database: "db".into(), target_schema: "public".into(), target_catalog: None, tables: vec!["a".into()], create_table: true, mode: TransferMode::Append, target_table_name_case: TransferTableNameCase::Preserve, quote_target_column_names: true, ownership_policy: TransferOwnershipPolicy::Preserve, batch_size: 1000, content: TransferContent::DataOnly, objects: Vec::new(), }; assert!(validate_transfer_request(&base).is_ok()); let with_objects = TransferRequest { objects: vec![TransferObjectSelection { object_type: TransferObjectKind::View, names: vec!["v".into()], }], ..base.clone() }; // DataOnly + objects → error assert!(validate_transfer_request(&with_objects).is_err()); let structure_only = TransferRequest { content: TransferContent::StructureOnly, ..base.clone() }; assert!(validate_transfer_request(&structure_only).is_ok()); } } mod transfer_existence_tests { use super::*; #[test] fn builds_target_existence_check_sql_per_family() { let mysql = target_object_exists_sql(&DatabaseType::Mysql, "shop", "v1", &TransferObjectKind::View).unwrap(); assert!(mysql.contains("information_schema.TABLES")); assert!(mysql.contains("TABLE_TYPE = 'VIEW'")); let mysql_ev = target_object_exists_sql(&DatabaseType::Mysql, "shop", "e1", &TransferObjectKind::Event).unwrap(); assert!(mysql_ev.contains("information_schema.EVENTS")); let mysql_tr = target_object_exists_sql(&DatabaseType::Mysql, "shop", "t1", &TransferObjectKind::Trigger).unwrap(); assert!(mysql_tr.contains("information_schema.TRIGGERS")); let pg = target_object_exists_sql(&DatabaseType::Postgres, "public", "v1", &TransferObjectKind::View).unwrap(); assert!(pg.contains("pg_class")); let orc = target_object_exists_sql(&DatabaseType::Oracle, "HR", "SEQ1", &TransferObjectKind::Sequence).unwrap(); assert!(orc.contains("ALL_OBJECTS")); assert!(target_object_exists_sql(&DatabaseType::Sqlite, "m", "x", &TransferObjectKind::View).is_err()); let ss_view = target_object_exists_sql(&DatabaseType::SqlServer, "dbo", "v1", &TransferObjectKind::View).unwrap(); assert!(ss_view.contains("sys.objects")); assert!(ss_view.contains("o.type IN ('V')")); let ss_seq = target_object_exists_sql(&DatabaseType::SqlServer, "dbo", "s1", &TransferObjectKind::Sequence).unwrap(); assert!(ss_seq.contains("o.type IN ('SO')")); let ss_trg = target_object_exists_sql(&DatabaseType::SqlServer, "dbo", "t1", &TransferObjectKind::Trigger).unwrap(); assert!(ss_trg.contains("o.type IN ('TR')")); } } mod transfer_cross_family_tests { use super::*; use TransferObjectKind::*; #[test] fn cross_family_transferable_kinds_matrix() { // cross-family VIEW transfer is disabled (the DDL conversion does // not translate the view query body, so source-specific constructs // could run unchanged on an incompatible target) assert!(cross_family_transferable_object_kinds(&DatabaseType::Mysql, &DatabaseType::Dameng).is_empty()); assert!(cross_family_transferable_object_kinds(&DatabaseType::Dameng, &DatabaseType::Mysql).is_empty()); assert!(cross_family_transferable_object_kinds(&DatabaseType::Mysql, &DatabaseType::SqlServer).is_empty()); // sqlserver <-> dameng: sequences only (plain DDL, converted and tested) assert_eq!( cross_family_transferable_object_kinds(&DatabaseType::SqlServer, &DatabaseType::Dameng), vec![Sequence] ); assert_eq!( cross_family_transferable_object_kinds(&DatabaseType::Dameng, &DatabaseType::SqlServer), vec![Sequence] ); // same family: all source kinds are transferable let same = cross_family_transferable_object_kinds(&DatabaseType::Mysql, &DatabaseType::Mysql); assert!(same.contains(&Trigger)); assert!(same.contains(&Event)); // unsupported databases assert!(cross_family_transferable_object_kinds(&DatabaseType::Sqlite, &DatabaseType::Mysql).is_empty()); assert_eq!(transfer_object_family(&DatabaseType::Gbase), Some(TransferObjectFamily::Mysql)); // postgres is not a validated cross-family source or target: // the executor rejects postgres sources and no dialect-aware // conversion exists for it (postgres <-> postgres stays same-family) assert!(cross_family_transferable_object_kinds(&DatabaseType::Postgres, &DatabaseType::Mysql).is_empty()); assert!(cross_family_transferable_object_kinds(&DatabaseType::Mysql, &DatabaseType::Postgres).is_empty()); assert!( cross_family_transferable_object_kinds(&DatabaseType::Postgres, &DatabaseType::SqlServer).is_empty() ); assert!(cross_family_transferable_object_kinds(&DatabaseType::Postgres, &DatabaseType::Postgres) .contains(&View)); } #[test] fn should_transfer_schema_objects_matrix() { // DataOnly never transfers schema objects, even for PG-family pairs. assert!(!should_transfer_schema_objects( &DatabaseType::Postgres, &DatabaseType::Postgres, &TransferContent::DataOnly, &[] )); assert!(!should_transfer_schema_objects( &DatabaseType::Postgres, &DatabaseType::Postgres, &TransferContent::DataOnly, &[TransferObjectSelection { object_type: TransferObjectKind::View, names: vec!["v1".into()] }] )); assert!(!should_transfer_schema_objects( &DatabaseType::Mysql, &DatabaseType::Dameng, &TransferContent::DataOnly, &[TransferObjectSelection { object_type: TransferObjectKind::View, names: vec!["v1".into()] }] )); // Non-empty selections participate in structure modes. assert!(should_transfer_schema_objects( &DatabaseType::Postgres, &DatabaseType::Mysql, &TransferContent::StructureOnly, &[TransferObjectSelection { object_type: TransferObjectKind::View, names: vec!["v1".into()] }] )); // Empty selection: PG→PG keeps the legacy transfer-everything default // only when structure participates in the transfer. assert!(should_transfer_schema_objects( &DatabaseType::Postgres, &DatabaseType::Postgres, &TransferContent::StructureOnly, &[] )); assert!(should_transfer_schema_objects( &DatabaseType::Kingbase, &DatabaseType::Postgres, &TransferContent::StructureAndData, &[] )); assert!(should_transfer_schema_objects( &DatabaseType::Postgres, &DatabaseType::OpenGauss, &TransferContent::StructureOnly, &[] )); // empty selection: every other combination transfers nothing assert!(!should_transfer_schema_objects( &DatabaseType::Postgres, &DatabaseType::Mysql, &TransferContent::StructureOnly, &[] )); assert!(!should_transfer_schema_objects( &DatabaseType::Mysql, &DatabaseType::Mysql, &TransferContent::StructureAndData, &[] )); assert!(!should_transfer_schema_objects( &DatabaseType::Mysql, &DatabaseType::Dameng, &TransferContent::StructureOnly, &[] )); assert!(!should_transfer_schema_objects( &DatabaseType::Dameng, &DatabaseType::SqlServer, &TransferContent::StructureAndData, &[] )); assert!(!should_transfer_schema_objects( &DatabaseType::Sqlite, &DatabaseType::Sqlite, &TransferContent::StructureOnly, &[] )); } #[test] fn rewrites_cross_family_view_ddl() { // mysql -> dameng let mysql_view = "CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`%` SQL SECURITY DEFINER VIEW `src`.`v` AS select `t`.`id` AS `id` from `src`.`t`"; let dm = convert_cross_family_object_ddl( &TransferObjectFamily::Mysql, &TransferObjectFamily::Oracle, &TransferObjectKind::View, "src", "TGT", mysql_view, ); assert!(dm.starts_with("CREATE VIEW \"TGT\".\"v\" AS"), "{dm}"); assert!(!dm.contains("DEFINER")); assert!(!dm.contains("ALGORITHM")); assert!(dm.contains("\"t\".\"id\"")); assert!(dm.contains("from \"TGT\".\"t\"")); // sqlserver -> mysql let ss_view = "CREATE VIEW [dbo].[v] WITH SCHEMABINDING AS SELECT a.id, b.name FROM [dbo].[a] JOIN [dbo].[b] ON a.id = b.id"; let my = convert_cross_family_object_ddl( &TransferObjectFamily::SqlServer, &TransferObjectFamily::Mysql, &TransferObjectKind::View, "dbo", "tgt", ss_view, ); assert!(my.starts_with("CREATE VIEW `tgt`.`v` AS"), "{my}"); assert!(!my.contains("SCHEMABINDING")); assert!(my.contains("`tgt`.`a`")); assert!(!my.contains("[")); // dameng -> sqlserver let dm_view = "CREATE OR REPLACE FORCE VIEW \"S\".\"V\" (\"ID\") AS SELECT \"ID\" FROM \"S\".\"T\""; let ss = convert_cross_family_object_ddl( &TransferObjectFamily::Oracle, &TransferObjectFamily::SqlServer, &TransferObjectKind::View, "S", "dbo", dm_view, ); assert!(ss.starts_with("CREATE VIEW [dbo].[V] ("), "{ss}"); assert!(!ss.contains("FORCE")); assert!(ss.contains("[dbo].[T]")); // dameng -> mysql let my2 = convert_cross_family_object_ddl( &TransferObjectFamily::Oracle, &TransferObjectFamily::Mysql, &TransferObjectKind::View, "S", "tgt", dm_view, ); assert!(my2.starts_with("CREATE VIEW `tgt`.`V` ("), "{my2}"); assert!(my2.contains("`tgt`.`T`")); } #[test] fn does_not_rewrite_identifiers_inside_strings_and_comments() { // MySQL -> Dameng: string literals and comments must not have // their content re-quoted or schema-qualified. let mysql_view = concat!( "CREATE DEFINER=`root`@`%` VIEW `v` AS ", "-- from `hidden`.`table` join \"hidden\"\n", "SELECT `t`.`id`, 'from \"lit\"', \"double\" ", "/* join \"quoted\" */ FROM `src`.`t` ", "WHERE `t`.`name` = 'join \"src\".\"t\"'", ); let dm = convert_cross_family_object_ddl( &TransferObjectFamily::Mysql, &TransferObjectFamily::Oracle, &TransferObjectKind::View, "src", "TGT", mysql_view, ); assert!(dm.starts_with("CREATE VIEW \"TGT\".\"v\" AS"), "{dm}"); // code identifiers are rewritten and qualified assert!(dm.contains("SELECT \"t\".\"id\""), "{dm}"); assert!(dm.contains("FROM \"TGT\".\"t\""), "{dm}"); // the comment is untouched assert!(dm.contains("-- from `hidden`.`table` join \"hidden\""), "{dm}"); assert!(dm.contains("/* join \"quoted\" */"), "{dm}"); // the single-quoted string literal is untouched (including the // fake qualifier inside it) assert!(dm.contains("'join \"src\".\"t\"'"), "{dm}"); // MySQL double-quoted text is a string literal, not an identifier assert!(dm.contains("\"double\""), "{dm}"); // SqlServer -> MySQL: brackets inside comments/strings stay put let ss_view = concat!( "CREATE VIEW [dbo].[v] AS ", "-- SELECT [dbo].[x]\n", "SELECT [a].[id], 'literal [dbo].[y]' FROM [dbo].[a]", ); let my = convert_cross_family_object_ddl( &TransferObjectFamily::SqlServer, &TransferObjectFamily::Mysql, &TransferObjectKind::View, "dbo", "tgt", ss_view, ); assert!(my.starts_with("CREATE VIEW `tgt`.`v` AS"), "{my}"); assert!(my.contains("`tgt`.`a`"), "{my}"); assert!(my.contains("-- SELECT [dbo].[x]"), "{my}"); assert!(my.contains("'literal [dbo].[y]'"), "{my}"); } #[test] fn sequence_rewrite_keeps_strings_intact() { // SqlServer -> Dameng: `AS BIGINT` stripping must not touch a // string literal that merely contains the token. let ss_seq = "CREATE SEQUENCE [dbo].[s] AS BIGINT START WITH 1 COMMENT 'AS BIGINT in comment'"; let dm = convert_cross_family_object_ddl( &TransferObjectFamily::SqlServer, &TransferObjectFamily::Oracle, &TransferObjectKind::Sequence, "dbo", "TGT", ss_seq, ); assert!(dm.starts_with("CREATE SEQUENCE \"TGT\".\"s\" START WITH 1"), "{dm}"); assert!(!dm.contains("\"s\" AS BIGINT"), "{dm}"); assert!(dm.contains("'AS BIGINT in comment'"), "{dm}"); } #[test] fn rewrites_cross_family_sequence_ddl() { // sqlserver -> dameng: strip AS type let ss_seq = "CREATE SEQUENCE [dbo].[seq1] AS BIGINT START WITH 5 INCREMENT BY 2 MINVALUE 1 MAXVALUE 1000 CACHE 50"; let dm = convert_cross_family_object_ddl( &TransferObjectFamily::SqlServer, &TransferObjectFamily::Oracle, &TransferObjectKind::Sequence, "dbo", "TGT", ss_seq, ); assert!(dm.starts_with("CREATE SEQUENCE \"TGT\".\"seq1\" START WITH 5"), "{dm}"); assert!(!dm.contains("AS BIGINT")); assert!(dm.contains("CACHE 50")); // dameng -> sqlserver: NO CYCLE spacing let dm_seq = "CREATE SEQUENCE \"S\".\"SEQ1\" START WITH 1 INCREMENT BY 1 MINVALUE 1 NOCYCLE NOCACHE"; let ss = convert_cross_family_object_ddl( &TransferObjectFamily::Oracle, &TransferObjectFamily::SqlServer, &TransferObjectKind::Sequence, "S", "dbo", dm_seq, ); assert!(ss.starts_with("CREATE SEQUENCE [dbo].[SEQ1]"), "{ss}"); assert!(ss.contains("NO CYCLE"), "{ss}"); assert!(ss.contains("NO CACHE"), "{ss}"); } #[test] fn rejects_unsupported_cross_family_objects_in_executor() { let kinds = cross_family_transferable_object_kinds(&DatabaseType::Mysql, &DatabaseType::SqlServer); // VIEW is disabled cross-family: only the DDL wrapper/quoting is // rewritten, the query body is not translated assert!(!kinds.contains(&View)); // procedures/triggers are never cross-family transferable assert!(!kinds.contains(&Procedure)); assert!(!kinds.contains(&Trigger)); assert!(!kinds.contains(&Event)); assert!(!kinds.contains(&Function)); } #[test] fn qualifies_cross_family_view_target_schema() { // mysql -> dameng: bare refs get the target schema prefix, prefixed // refs keep their (rewritten) prefix let mysql_view = "CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`%` SQL SECURITY DEFINER VIEW `v` AS select a.id from `exam_record` a join `exam`.`t2` t on t.id = a.id"; let dm = convert_cross_family_object_ddl( &TransferObjectFamily::Mysql, &TransferObjectFamily::Oracle, &TransferObjectKind::View, "exam", "DM", mysql_view, ); assert!(dm.starts_with("CREATE VIEW \"DM\".\"v\" AS"), "{dm}"); assert!(dm.contains("from \"DM\".\"exam_record\" a"), "{dm}"); assert!(dm.contains("join \"DM\".\"t2\" t"), "{dm}"); // dameng -> mysql: quoted prefix style switches to backticks let dm_view = "CREATE OR REPLACE FORCE NONEDITIONABLE VIEW \"DM\".\"v\" AS select id from \"DM\".\"exam_record\""; let my = convert_cross_family_object_ddl( &TransferObjectFamily::Oracle, &TransferObjectFamily::Mysql, &TransferObjectKind::View, "DM", "exam", dm_view, ); assert!(my.starts_with("CREATE VIEW `exam`.`v` AS"), "{my}"); assert!(my.contains("from `exam`.`exam_record`"), "{my}"); // sqlserver -> dameng let ss_view = "CREATE VIEW [dbo].[v] WITH SCHEMABINDING AS select id from [dbo].[exam_record]"; let dm2 = convert_cross_family_object_ddl( &TransferObjectFamily::SqlServer, &TransferObjectFamily::Oracle, &TransferObjectKind::View, "dbo", "TGT", ss_view, ); assert!(dm2.starts_with("CREATE VIEW \"TGT\".\"v\" AS"), "{dm2}"); assert!(dm2.contains("from \"TGT\".\"exam_record\""), "{dm2}"); } #[test] fn qualifies_cross_family_view_with_cjk_identifiers() { // mysql -> dameng with Chinese table/view names let mysql_view = "CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`%` SQL SECURITY DEFINER VIEW `成绩排名` AS select a.id from `用户表` a join `exam`.`成绩明细` t on t.id = a.id"; let dm = convert_cross_family_object_ddl( &TransferObjectFamily::Mysql, &TransferObjectFamily::Oracle, &TransferObjectKind::View, "exam", "DM", mysql_view, ); assert!(dm.starts_with("CREATE VIEW \"DM\".\"成绩排名\" AS"), "{dm}"); assert!(dm.contains("from \"DM\".\"用户表\" a"), "{dm}"); assert!(dm.contains("join \"DM\".\"成绩明细\" t"), "{dm}"); } } mod transfer_mysql_ddl_tests { use super::*; #[test] fn strips_mysql_definer_clauses() { assert_eq!(strip_mysql_definer("CREATE DEFINER=`u`@`%` VIEW v AS SELECT 1"), "CREATE VIEW v AS SELECT 1"); assert_eq!( strip_mysql_definer("CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` VIEW v AS SELECT 1"), "CREATE ALGORITHM=UNDEFINED VIEW v AS SELECT 1" ); assert_eq!(strip_mysql_definer("CREATE PROCEDURE p() BEGIN END"), "CREATE PROCEDURE p() BEGIN END"); } #[test] fn rewrites_mysql_schema_qualifiers() { assert_eq!( rewrite_mysql_schema_qualifier("CREATE VIEW `src`.`v` AS SELECT 1 FROM `src`.`t`", "src", "dst"), "CREATE VIEW `dst`.`v` AS SELECT 1 FROM `dst`.`t`" ); } #[test] fn assembles_mysql_trigger_and_event_ddl() { let trigger = mysql_trigger_ddl("shop", "trg1", "BEFORE", "INSERT", "users", "SET NEW.updated = NOW()"); assert_eq!( trigger, "CREATE TRIGGER `trg1` BEFORE INSERT ON `shop`.`users` FOR EACH ROW SET NEW.updated = NOW()" ); let event = mysql_event_ddl("shop", "ev1", "ENABLE", "EVERY 1 DAY", "DELETE FROM logs"); assert_eq!(event, "CREATE EVENT `ev1` ON SCHEDULE EVERY 1 DAY ENABLE DO DELETE FROM logs"); } #[test] fn deferred_mysql_foreign_keys_follow_target_ddl_capability() { for target in [DatabaseType::Mysql, DatabaseType::StarRocks, DatabaseType::Goldendb, DatabaseType::Sundb] { assert!(supports_deferred_mysql_foreign_keys(&target), "{target:?}"); } assert!(!supports_deferred_mysql_foreign_keys(&DatabaseType::Doris)); } #[test] fn strips_inline_foreign_keys_from_mysql_create_table() { let ddl = "CREATE TABLE `child` (\n `id` int NOT NULL,\n `parent_id` int NOT NULL,\n PRIMARY KEY (`id`),\n KEY `fk_child_parent` (`parent_id`),\n CONSTRAINT `fk_child_parent` FOREIGN KEY (`parent_id`) REFERENCES `parent` (`id`)\n) ENGINE=InnoDB"; let stripped = strip_inline_foreign_key_constraint_lines(ddl); assert!(!stripped.to_ascii_uppercase().contains("FOREIGN KEY"), "{stripped}"); assert!(stripped.contains("KEY `fk_child_parent` (`parent_id`)"), "{stripped}"); } #[test] fn keeps_columns_whose_comment_mentions_foreign_key() { // Regression for #7660: the deferred-FK DDL rewrite used to drop any // line containing " FOREIGN KEY ", so column definitions whose // COMMENT text merely mentions the words silently vanished from the // created target table. let ddl = "CREATE TABLE `title_task` (\n `id` varchar(32) NOT NULL,\n `review_result` varchar(100) DEFAULT NULL,\n `review_mode` tinyint NOT NULL DEFAULT '0' COMMENT 'foreign key of review flow',\n `score_snapshot_json` text COMMENT 'snapshot json, see foreign key docs',\n `task_status` int DEFAULT '0',\n PRIMARY KEY (`id`),\n KEY `idx_task_status` (`task_status`),\n CONSTRAINT `fk_title_task_org` FOREIGN KEY (`org_id`) REFERENCES `org` (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4"; let stripped = strip_inline_foreign_key_constraint_lines(ddl); assert!(stripped.contains("`review_mode` tinyint NOT NULL DEFAULT '0'"), "{stripped}"); assert!(stripped.contains("`score_snapshot_json` text"), "{stripped}"); assert!(!stripped.contains("FOREIGN KEY"), "{stripped}"); assert!(stripped.contains("KEY `idx_task_status` (`task_status`)"), "{stripped}"); } #[test] fn strips_only_foreign_key_constraint_lines_regardless_of_other_mentions() { // A CHECK constraint whose expression mentions the words stays; a // nameless inline `FOREIGN KEY (` line (defensive: some MySQL // dialects emit it) is still stripped. let ddl = "CREATE TABLE `t` (\n `id` int NOT NULL,\n `kind` varchar(10) NOT NULL,\n CONSTRAINT `chk_kind` CHECK (`kind` <> 'foreign key'),\n FOREIGN KEY (`id`) REFERENCES `parent` (`id`)\n) ENGINE=InnoDB"; let stripped = strip_inline_foreign_key_constraint_lines(ddl); assert!(stripped.contains("CONSTRAINT `chk_kind` CHECK (`kind` <> 'foreign key')"), "{stripped}"); assert!(!stripped.contains("REFERENCES"), "{stripped}"); } #[test] fn generates_deferred_mysql_foreign_key_alter_statements() { let foreign_keys = vec![db::ForeignKeyInfo { name: "fk_child_parent".to_string(), column: "parent_id".to_string(), ref_schema: None, ref_table: "parent".to_string(), ref_column: "id".to_string(), on_update: None, on_delete: Some("CASCADE".to_string()), }]; let request = test_transfer_request(vec!["child", "parent"]); let statements = generate_mysql_foreign_key_alter_statements(&foreign_keys, &request, "child", &DatabaseType::Mysql); assert_eq!( statements, vec![ "ALTER TABLE `child` ADD CONSTRAINT `fk_child_parent` FOREIGN KEY (`parent_id`) REFERENCES `parent` (`id`) ON DELETE CASCADE" .to_string() ] ); } #[test] fn same_database_mysql_foreign_key_applies_target_table_name_rules() { // test_transfer_request's source_schema is "source_schema" — the // referenced table's ref_schema matching that exactly is what marks it // as part of this transfer batch (see mysql_table_metadata_catalog-style // fallback in generate_mysql_foreign_key_alter_statements). let foreign_keys = vec![db::ForeignKeyInfo { name: "fk_child_parent".to_string(), column: "parent_id".to_string(), ref_schema: Some("source_schema".to_string()), ref_table: "Parent".to_string(), ref_column: "id".to_string(), on_update: None, on_delete: None, }]; let mut request = test_transfer_request(vec!["child", "Parent"]); request.target_table_name_case = TransferTableNameCase::Lower; let statements = generate_mysql_foreign_key_alter_statements(&foreign_keys, &request, "child", &DatabaseType::Mysql); // Referenced table is in-batch, so its target-side name (lowercased per // target_table_name_case) is used, unqualified — it lives in whatever // database this ALTER TABLE already runs against. assert_eq!( statements, vec![ "ALTER TABLE `child` ADD CONSTRAINT `fk_child_parent` FOREIGN KEY (`parent_id`) REFERENCES `parent` (`id`)" .to_string() ] ); } #[test] fn cross_database_mysql_foreign_key_keeps_original_schema_and_name() { let foreign_keys = vec![db::ForeignKeyInfo { name: "fk_child_parent".to_string(), column: "parent_id".to_string(), ref_schema: Some("other_db".to_string()), ref_table: "Parent".to_string(), ref_column: "id".to_string(), on_update: None, on_delete: None, }]; let mut request = test_transfer_request(vec!["child"]); // Even with a target-side rename policy configured, a table outside the // transfer batch (different database) must not be renamed — we never // created or renamed it, so it must be referenced exactly as it exists // on the target server. request.target_table_name_case = TransferTableNameCase::Lower; let statements = generate_mysql_foreign_key_alter_statements(&foreign_keys, &request, "child", &DatabaseType::Mysql); assert_eq!( statements, vec![ "ALTER TABLE `child` ADD CONSTRAINT `fk_child_parent` FOREIGN KEY (`parent_id`) REFERENCES `other_db`.`Parent` (`id`)" .to_string() ] ); } #[test] fn group_foreign_keys_by_constraint_name_preserves_first_seen_order() { let foreign_keys = vec![ db::ForeignKeyInfo { name: "fk_b".to_string(), column: "b1".to_string(), ref_schema: None, ref_table: "t2".to_string(), ref_column: "id".to_string(), on_update: None, on_delete: None, }, db::ForeignKeyInfo { name: "fk_a".to_string(), column: "a1".to_string(), ref_schema: None, ref_table: "t3".to_string(), ref_column: "id".to_string(), on_update: None, on_delete: None, }, // Second column of the same multi-column fk_a constraint — must be // grouped with the first, not treated as a new constraint. db::ForeignKeyInfo { name: "fk_a".to_string(), column: "a2".to_string(), ref_schema: None, ref_table: "t3".to_string(), ref_column: "id2".to_string(), on_update: None, on_delete: None, }, ]; let grouped = group_foreign_keys_by_constraint_name(&foreign_keys); assert_eq!(grouped.len(), 2); assert_eq!(grouped[0].0, "fk_b"); assert_eq!(grouped[0].1.len(), 1); assert_eq!(grouped[1].0, "fk_a"); assert_eq!(grouped[1].1.len(), 2); assert_eq!(grouped[1].1[0].column, "a1"); assert_eq!(grouped[1].1[1].column, "a2"); } #[test] fn foreign_key_cycle_survives_dependency_sort_via_deferred_alters() { // A <-> B mutual reference has no valid CREATE TABLE order at all — the // dependency sort can only push both to the back (see // table_dependency_sort_ignores_duplicates_and_out_of_scope_tables-style // cycle handling); the transfer must not depend on ordering to succeed, // only on foreign keys being added after every table exists. let tables = vec!["b_department".to_string(), "c_employee".to_string()]; let dependencies = vec![ ("b_department".to_string(), "c_employee".to_string()), ("c_employee".to_string(), "b_department".to_string()), ]; let sorted = sort_table_names_by_dependencies(&tables, &dependencies, true); // Whatever order the sort settles on, both tables are present — proving // table creation alone can proceed regardless of the cycle, which is the // property the deferred-ALTER approach relies on. assert_eq!(sorted.len(), 2); assert!(sorted.contains(&"b_department".to_string())); assert!(sorted.contains(&"c_employee".to_string())); } } mod transfer_sqlserver_source_tests { use super::*; use serde_json::json; #[test] fn builds_sqlserver_object_source_queries() { let view = sqlserver_object_source_query(&TransferObjectKind::View, "dbo", "v1").unwrap(); assert!(view.contains("sys.sql_modules"), "{view}"); assert!(view.contains("o.type IN ('V')"), "{view}"); let routine = sqlserver_object_source_query(&TransferObjectKind::Procedure, "dbo", "p1").unwrap(); assert!(routine.contains("o.type IN ('P')"), "{routine}"); let trigger = sqlserver_object_source_query(&TransferObjectKind::Trigger, "dbo", "t1").unwrap(); assert!(trigger.contains("o.type IN ('TR')"), "{trigger}"); let seq = sqlserver_object_source_query(&TransferObjectKind::Sequence, "dbo", "s1").unwrap(); assert!(seq.contains("sys.sequences"), "{seq}"); assert!(!seq.contains("sys.sql_modules"), "{seq}"); } #[test] fn extracts_sqlserver_object_ddl_from_result() { let view_result = test_query_result(vec![vec![json!("CREATE VIEW [dbo].[v1] AS SELECT 1 AS x")]]); let ddl = sqlserver_object_ddl_from_result(&view_result, "dbo", "v1", &TransferObjectKind::View).unwrap(); assert_eq!(ddl, "CREATE VIEW [dbo].[v1] AS SELECT 1 AS x"); // start, increment, min, max, cycle, cache let seq_result = test_query_result(vec![vec![ json!("1"), json!("2"), json!("5"), json!("1000"), json!("NO CYCLE"), json!("50"), ]]); let seq_ddl = sqlserver_object_ddl_from_result(&seq_result, "dbo", "s1", &TransferObjectKind::Sequence).unwrap(); assert!( seq_ddl.starts_with("CREATE SEQUENCE [dbo].[s1] START WITH 1 INCREMENT BY 2 MINVALUE 5 MAXVALUE 1000"), "{seq_ddl}" ); assert!(seq_ddl.contains("NO CYCLE"), "{seq_ddl}"); assert!(seq_ddl.contains("CACHE 50"), "{seq_ddl}"); } } mod transfer_mysql_source_tests { use super::*; #[test] fn builds_mysql_object_source_queries() { let sql = mysql_object_source_query(&TransferObjectKind::View, "shop", "v1").unwrap(); assert!(sql.contains("SHOW CREATE VIEW")); let sql = mysql_object_source_query(&TransferObjectKind::Trigger, "shop", "trg1").unwrap(); assert!(sql.contains("information_schema.TRIGGERS")); assert!(sql.contains("TRIGGER_NAME = 'trg1'")); let sql = mysql_object_source_query(&TransferObjectKind::Event, "shop", "ev1").unwrap(); assert!(sql.contains("information_schema.EVENTS")); assert!(sql.contains("EVENT_NAME = 'ev1'")); } #[test] fn extracts_mysql_object_ddl_from_result() { let view = mysql_object_ddl_from_result( &TransferObjectKind::View, "shop", &[vec![serde_json::json!("v1"), serde_json::json!("CREATE VIEW `shop`.`v1` AS SELECT 1")]], ) .unwrap(); assert_eq!(view, "CREATE VIEW `shop`.`v1` AS SELECT 1"); let routine = mysql_object_ddl_from_result( &TransferObjectKind::Procedure, "shop", &[vec![ serde_json::json!("p"), serde_json::json!("PROCEDURE"), serde_json::json!("CREATE PROCEDURE p() BEGIN END"), ]], ) .unwrap(); assert_eq!(routine, "CREATE PROCEDURE p() BEGIN END"); let trigger = mysql_object_ddl_from_result( &TransferObjectKind::Trigger, "shop", &[vec![ serde_json::json!("trg1"), serde_json::json!("BEFORE"), serde_json::json!("INSERT"), serde_json::json!("users"), serde_json::json!("SET NEW.updated = NOW()"), ]], ) .unwrap(); assert_eq!( trigger, "CREATE TRIGGER `trg1` BEFORE INSERT ON `shop`.`users` FOR EACH ROW SET NEW.updated = NOW()" ); let event = mysql_object_ddl_from_result( &TransferObjectKind::Event, "shop", &[vec![ serde_json::json!("ev1"), serde_json::json!("ENABLE"), serde_json::json!("2026-01-01"), serde_json::json!("1"), serde_json::json!("DAY"), serde_json::json!("DELETE FROM logs"), ]], ) .unwrap(); assert_eq!(event, "CREATE EVENT `ev1` ON SCHEDULE EVERY 1 DAY ENABLE DO DELETE FROM logs"); } } mod transfer_oracle_source_tests { use super::*; #[test] fn builds_oracle_object_source_query() { let sql = oracle_object_source_query(&TransferObjectKind::Trigger, "HR", "TRG1").unwrap(); assert!(sql.contains("DBMS_METADATA.GET_DDL('TRIGGER', 'TRG1', 'HR')")); let sql = oracle_object_source_query(&TransferObjectKind::Sequence, "HR", "SEQ1").unwrap(); assert!(sql.contains("DBMS_METADATA.GET_DDL('SEQUENCE', 'SEQ1', 'HR')")); let sql = oracle_object_source_query(&TransferObjectKind::View, "", "V1").unwrap(); assert!(!sql.contains(",'")); } #[test] fn rewrites_oracle_schema_qualifiers() { let ddl = concat!( "CREATE OR REPLACE TRIGGER \"HR\".\"TRG1\" ... '\"HR\".literal';\n", "-- keep \"HR\".line_comment\n", "/* keep \"HR\".block_comment */", ); assert_eq!( rewrite_oracle_schema_qualifier(ddl, "HR", "APP"), concat!( "CREATE OR REPLACE TRIGGER \"APP\".\"TRG1\" ... '\"HR\".literal';\n", "-- keep \"HR\".line_comment\n", "/* keep \"HR\".block_comment */", ) ); } } mod transfer_executor_tests { use super::*; #[test] fn orders_object_selections_by_dependency() { let kinds = vec![ TransferObjectKind::Trigger, TransferObjectKind::View, TransferObjectKind::Sequence, TransferObjectKind::Event, TransferObjectKind::Procedure, TransferObjectKind::Function, ]; let ordered = ordered_transfer_object_kinds(kinds); assert_eq!(ordered[0], TransferObjectKind::Sequence); assert_eq!(ordered[1], TransferObjectKind::View); assert_eq!(ordered[2], TransferObjectKind::Function); assert_eq!(ordered[3], TransferObjectKind::Procedure); assert_eq!(ordered[4], TransferObjectKind::Trigger); assert_eq!(ordered[5], TransferObjectKind::Event); } } mod transfer_mysql_executor_tests { use super::*; #[test] fn extracts_selected_names_by_kind() { let selections = vec![ TransferObjectSelection { object_type: TransferObjectKind::View, names: vec!["v1".into()] }, TransferObjectSelection { object_type: TransferObjectKind::View, names: vec!["v2".into()] }, TransferObjectSelection { object_type: TransferObjectKind::Trigger, names: vec!["t1".into()] }, ]; let views = selected_object_names(&selections, &TransferObjectKind::View); assert_eq!(views, vec!["v1", "v2"]); assert!(selected_object_names(&selections, &TransferObjectKind::Event).is_empty()); } } mod transfer_oracle_executor_tests { use super::*; #[test] fn resolves_oracle_family_source_schema() { assert_eq!(resolve_oracle_schema("", "HR"), "HR"); assert_eq!(resolve_oracle_schema("HR", "db"), "HR"); } } mod transfer_postgres_executor_tests { use super::*; #[test] fn postgres_transfer_catalog_probe_checks_legacy_boundaries() { let sql = postgres_transfer_catalog_capabilities_sql(); assert!(sql.contains("attrelid = 'pg_catalog.pg_proc'::regclass")); assert!(sql.contains("attname = 'prokind'")); assert!(sql.contains("NOT attisdropped")); assert!(sql.contains("c.relname = 'pg_policy'")); assert!(sql.contains("attname = 'polpermissive'")); assert!(!sql.contains("'pg_catalog.pg_policy'::regclass")); } #[test] fn postgres_table_transfer_reuses_the_batch_schema_preflight() { assert!(!transfer_table_needs_inline_postgres_schema_ensure( &DatabaseType::Postgres, &DatabaseType::Postgres )); assert!(transfer_table_needs_inline_postgres_schema_ensure( &DatabaseType::Postgres, &DatabaseType::Kingbase )); } #[test] fn postgres_transfer_routine_sources_support_legacy_catalogs() { let modern = postgres_transfer_routines_sql("public", true); assert!(modern.contains("CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END")); assert!(modern.contains("p.prokind IN ('p','f')")); assert!(!modern.contains("p.proisagg")); let legacy = postgres_transfer_routines_sql("public", false); assert!(!legacy.contains("prokind")); assert!(legacy.contains("'FUNCTION'")); assert!(legacy.contains("NOT p.proisagg")); assert!(legacy.contains("NOT p.proiswindow")); } #[test] fn postgres_transfer_ownership_supports_legacy_catalogs() { let modern = postgres_transfer_ownership_statements_sql("public", "archive", &["items".into()], true); assert!(modern.contains("CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END")); assert!(modern.contains("p.prokind IN ('p','f')")); let legacy = postgres_transfer_ownership_statements_sql("public", "archive", &["items".into()], false); assert!(!legacy.contains("prokind")); assert!(legacy.contains("'FUNCTION'")); assert!(legacy.contains("NOT p.proisagg")); assert!(legacy.contains("NOT p.proiswindow")); } #[test] fn postgres_transfer_grants_support_legacy_catalogs() { let modern = postgres_transfer_grant_statements_sql("public", "archive", &["items".into()], true); assert!(modern.contains("CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END")); assert!(modern.contains("p.prokind IN ('p','f')")); let legacy = postgres_transfer_grant_statements_sql("public", "archive", &["items".into()], false); assert!(!legacy.contains("prokind")); assert!(legacy.contains("'FUNCTION'::text AS routine_kind")); assert!(legacy.contains("NOT p.proisagg")); assert!(legacy.contains("NOT p.proiswindow")); assert!(!legacy.contains("LATERAL")); assert!(legacy.contains("(aclexplode(n.nspacl)).*")); assert!(legacy.contains("(aclexplode(c.relacl)).*")); assert!(legacy.contains("(aclexplode(p.proacl)).*")); } #[test] fn filters_postgres_object_sources_by_selection() { let sources = vec![ db::ObjectSource { name: "v1".into(), object_type: db::ObjectSourceKind::View, schema: Some("public".into()), source: "SELECT 1".into(), editable: None, }, db::ObjectSource { name: "v2".into(), object_type: db::ObjectSourceKind::View, schema: Some("public".into()), source: "SELECT 2".into(), editable: None, }, ]; let selection = vec![TransferObjectSelection { object_type: TransferObjectKind::View, names: vec!["v1".into()] }]; let filtered = filter_object_sources_by_selection(sources, &selection); assert_eq!(filtered.len(), 1); assert_eq!(filtered[0].name, "v1"); } #[test] fn selected_postgres_sequences_are_prepared_without_changing_default_requests() { let mut request = test_transfer_request(vec!["biz_banner"]); assert!(selected_postgres_sequence_names(&request).is_empty()); request.objects = vec![ TransferObjectSelection { object_type: TransferObjectKind::Table, names: vec!["biz_banner".into()] }, TransferObjectSelection { object_type: TransferObjectKind::Sequence, names: vec!["biz_banner_id_seq".into(), "biz_banner_id_seq".into()], }, ]; assert_eq!(selected_postgres_sequence_names(&request), vec!["biz_banner_id_seq"]); assert_eq!(postgres_transfer_relation_names(&request), vec!["biz_banner", "biz_banner_id_seq"]); let sql = postgres_selected_sequences_sql("public", &selected_postgres_sequence_names(&request)).unwrap(); assert!(sql.contains("c.relname IN ('biz_banner_id_seq')")); assert!(sql.contains("pg_sequence_last_value(c.oid)::text")); } #[test] fn postgres_owned_sequence_queries_support_pre_ten_catalogs() { assert!(!POSTGRES_OWNED_SEQUENCES_SQL.contains("pg_sequence")); assert!(!POSTGRES_SEQUENCE_SNAPSHOTS_SQL.contains("pg_sequence")); for sql in [POSTGRES_OWNED_SEQUENCES_SQL, POSTGRES_SEQUENCE_SNAPSHOTS_SQL] { assert!(sql.contains("c.relkind = 'S'")); assert!(sql.contains("pg_depend")); assert!(sql.contains("d.deptype IN ('a', 'i')")); } } #[test] fn postgres_selected_sequence_ddl_preserves_definition_and_value() { let sequence = PostgresTransferSequence { name: "biz_banner_id_seq".into(), data_type: "bigint".into(), start_value: "5".into(), min_value: "-10".into(), max_value: "999".into(), increment: "2".into(), cycle: true, cache_value: "7".into(), last_value: Some("41".into()), }; assert_eq!( generate_postgres_transfer_sequence_create_ddl(&sequence, "archive"), "CREATE SEQUENCE IF NOT EXISTS \"archive\".\"biz_banner_id_seq\"\n AS bigint\n START WITH 5\n INCREMENT BY 2\n MINVALUE -10\n MAXVALUE 999\n CACHE 7\n CYCLE" ); assert_eq!( generate_postgres_transfer_sequence_setval_sql(&sequence, "archive"), Some("SELECT setval('\"archive\".\"biz_banner_id_seq\"', 41, true)".into()) ); let never_called = PostgresTransferSequence { last_value: None, ..sequence }; assert_eq!(generate_postgres_transfer_sequence_setval_sql(&never_called, "archive"), None); } } mod transfer_content_mode_tests { use super::*; #[test] fn structure_only_skips_data_steps() { assert!(should_copy_data(&TransferContent::StructureAndData)); assert!(should_copy_data(&TransferContent::DataOnly)); assert!(!should_copy_data(&TransferContent::StructureOnly)); } } fn test_transfer_request(tables: Vec<&str>) -> TransferRequest { TransferRequest { transfer_id: "transfer-1".to_string(), source_connection_id: "source".to_string(), source_database: "source_db".to_string(), source_schema: "source_schema".to_string(), source_catalog: None, target_connection_id: "target".to_string(), target_database: "target_db".to_string(), target_schema: "target_schema".to_string(), target_catalog: None, tables: tables.into_iter().map(str::to_string).collect(), create_table: true, content: TransferContent::default(), objects: Vec::new(), mode: TransferMode::Append, target_table_name_case: TransferTableNameCase::Preserve, quote_target_column_names: true, ownership_policy: TransferOwnershipPolicy::Preserve, batch_size: 1000, } } #[test] fn transfer_request_defaults_preserve_table_name_case() { let request: TransferRequest = serde_json::from_value(json!({ "transferId": "transfer-1", "sourceConnectionId": "source", "sourceDatabase": "source_db", "sourceSchema": "source_schema", "targetConnectionId": "target", "targetDatabase": "target_db", "targetSchema": "target_schema", "tables": ["ORDERS"], "createTable": true, "mode": "append", "batchSize": 1000 })) .unwrap(); assert_eq!(request.target_table_name_case, TransferTableNameCase::Preserve); assert_eq!(request.target_table_name("ORDERS"), "ORDERS"); } #[test] fn transfer_existing_target_table_name_prefers_exact_case() { let tables = vec![test_table("orders"), test_table("Orders")]; assert_eq!(existing_transfer_target_table_name("Orders", &tables, true), Some("Orders".to_string())); } #[test] fn transfer_existing_target_table_name_respects_case_sensitive_targets() { let tables = vec![test_table("Orders")]; assert_eq!(existing_transfer_target_table_name("orders", &tables, false), None); assert_eq!(existing_transfer_target_table_name("orders", &tables, true), Some("Orders".to_string())); } #[test] fn transfer_existing_target_table_name_ignores_contains_matches() { let tables = vec![test_table("archived_orders"), test_table("orders_backup")]; assert_eq!(existing_transfer_target_table_name("orders", &tables, true), None); } #[test] fn parses_mysql_lower_case_table_names_values() { let string_result = test_query_result(vec![vec![json!("lower_case_table_names"), json!("2")]]); let numeric_result = test_query_result(vec![vec![json!("lower_case_table_names"), json!(1)]]); let empty_result = test_query_result(Vec::new()); assert_eq!(mysql_lower_case_table_names_from_result(&string_result), Some(2)); assert_eq!(mysql_lower_case_table_names_from_result(&numeric_result), Some(1)); assert_eq!(mysql_lower_case_table_names_from_result(&empty_result), None); } #[test] fn transfer_table_name_case_transforms_target_names() { let mut request = test_transfer_request(vec!["ORDERS"]); request.target_table_name_case = TransferTableNameCase::Lower; assert_eq!(request.target_table_name("ORDERS"), "orders"); request.target_table_name_case = TransferTableNameCase::Upper; assert_eq!(request.target_table_name("orders"), "ORDERS"); } #[test] fn transfer_table_name_case_detects_target_collisions() { let mut request = test_transfer_request(vec!["ORDERS", "orders"]); request.target_table_name_case = TransferTableNameCase::Lower; let error = validate_transfer_target_table_names(&request).unwrap_err(); assert!(error.contains("both map to 'orders'")); } #[test] fn detects_identity_extras_for_selected_columns() { assert!(selected_columns_include_identity_extras( &[String::from("id"), String::from("name")], &[Some(String::from("identity")), None], )); assert!(selected_columns_include_identity_extras( &[String::from("id")], &[Some(String::from("auto_increment"))], )); assert!(!selected_columns_include_identity_extras( &[String::from("name")], &[None, Some(String::from("identity"))], )); } #[test] fn detects_selected_identity_columns_from_target_metadata() { let target_columns = vec![ db::ColumnInfo { extra: Some("identity".to_string()), ..test_column("ID", "INT") }, test_column("NAME", "VARCHAR(20)"), ]; assert!(selected_columns_include_identity_columns(&[String::from("id")], &target_columns)); assert!(!selected_columns_include_identity_columns(&[String::from("name")], &target_columns)); } #[test] fn detects_selected_postgres_generated_always_identity_columns() { let target_columns = vec![ db::ColumnInfo { name: "ID".to_string(), extra: Some(" GeNeRaTeD\tALWAYS AS\nIDENTITY (start with 1 increment by 1)".to_string()), ..test_column("ID", "bigint") }, db::ColumnInfo { extra: Some("generated by default as identity".to_string()), ..test_column("by_default_id", "bigint") }, db::ColumnInfo { extra: Some("generated always as (quantity * 2) stored".to_string()), ..test_column("total", "bigint") }, test_column("name", "text"), ]; assert!(selected_columns_include_postgres_generated_always_identity_columns( &[String::from("id")], &target_columns )); assert!(!selected_columns_include_postgres_generated_always_identity_columns( &[String::from("by_default_id")], &target_columns )); assert!(!selected_columns_include_postgres_generated_always_identity_columns( &[String::from("total")], &target_columns )); assert!(!selected_columns_include_postgres_generated_always_identity_columns( &[String::from("name")], &target_columns )); } #[test] fn sqlserver_writable_transfer_columns_skip_rowversion_types() { let columns = vec![ test_column("id", "int"), test_column("TimeSpan", "timestamp"), test_column("rv", "ROWVERSION"), test_column("name", "nvarchar(64)"), ]; let writable = writable_transfer_columns(&columns, &DatabaseType::SqlServer, &DatabaseType::SqlServer); assert_eq!(writable.iter().map(|column| column.name.as_str()).collect::>(), vec!["id", "name"]); } #[test] fn mysql_writable_transfer_columns_skip_only_generated_columns() { let columns = vec![ test_column("id", "int"), db::ColumnInfo { extra: Some("DEFAULT_GENERATED".to_string()), ..test_column("created_at", "timestamp") }, db::ColumnInfo { extra: Some("auto_increment".to_string()), ..test_column("sequence_id", "bigint") }, db::ColumnInfo { extra: Some("VIRTUAL GENERATED".to_string()), ..test_column("virtual_total", "decimal(10,2)") }, db::ColumnInfo { extra: Some("stored generated".to_string()), ..test_column("stored_hash", "varchar(64)") }, db::ColumnInfo { extra: Some("PERSISTENT GENERATED".to_string()), ..test_column("persistent_total", "decimal(10,2)") }, db::ColumnInfo { extra: Some("GENERATED ALWAYS".to_string()), ..test_column("explicit_generated", "int") }, db::ColumnInfo { extra: Some("on update CURRENT_TIMESTAMP".to_string()), ..test_column("updated_at", "timestamp") }, ]; let writable = writable_transfer_columns(&columns, &DatabaseType::Mysql, &DatabaseType::Mysql); assert_eq!( writable.iter().map(|column| column.name.as_str()).collect::>(), vec!["id", "created_at", "sequence_id", "updated_at"] ); assert_eq!(columns.len(), 8, "DDL metadata must retain generated columns"); } #[test] fn non_mysql_transfer_columns_keep_generated_markers() { let columns = vec![ test_column("id", "int"), db::ColumnInfo { extra: Some("STORED GENERATED".to_string()), ..test_column("computed", "int") }, ]; let writable = writable_transfer_columns(&columns, &DatabaseType::Postgres, &DatabaseType::Postgres); assert_eq!(writable.iter().map(|column| column.name.as_str()).collect::>(), vec!["id", "computed"]); } #[test] fn non_sqlserver_target_writable_transfer_columns_keep_timestamp_type() { let columns = vec![test_column("id", "int"), test_column("updated_at", "timestamp")]; let writable = writable_transfer_columns(&columns, &DatabaseType::Postgres, &DatabaseType::Postgres); assert_eq!(writable.iter().map(|column| column.name.as_str()).collect::>(), vec!["id", "updated_at"]); } #[test] fn sqlserver_target_keeps_timestamp_from_other_source_databases() { let columns = vec![test_column("id", "int"), test_column("updated_at", "timestamp")]; let writable = writable_transfer_columns(&columns, &DatabaseType::Postgres, &DatabaseType::SqlServer); assert_eq!(writable.iter().map(|column| column.name.as_str()).collect::>(), vec!["id", "updated_at"]); } #[test] fn transfer_target_column_validation_reports_columns_absent_from_target() { let target_columns = vec![test_column("id", "int"), test_column("name", "varchar(32)")]; let col_names = vec!["id".to_string(), "name".to_string(), "extra_col".to_string()]; assert_eq!( missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Mysql, true), vec!["extra_col".to_string()] ); } #[test] fn transfer_target_column_validation_uses_database_case_rules() { let target_columns = vec![test_column("ID", "int"), test_column("Name", "varchar(32)")]; let col_names = vec!["id".to_string(), "name".to_string()]; assert!(missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Mysql, true).is_empty()); assert!(missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Kyuubi, true).is_empty()); assert_eq!( missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Postgres, true), vec!["id".to_string(), "name".to_string()] ); } #[test] fn gauss_target_column_validation_ignores_case_when_quoting_disabled() { let target_columns = vec![test_column("id", "int"), test_column("user_id", "bigint")]; let col_names = vec!["id".to_string(), "USER_ID".to_string()]; assert!(missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Gaussdb, false).is_empty()); assert!( missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::OpenGauss, false).is_empty() ); assert_eq!( missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Gaussdb, true), vec!["USER_ID".to_string()] ); } #[test] fn transfer_target_column_validation_allows_omittable_target_columns() { let target_columns = vec![ test_column("id", "int"), test_column("nullable_note", "varchar(32)"), db::ColumnInfo { is_nullable: false, column_default: Some("CURRENT_TIMESTAMP".to_string()), ..test_column("created_at", "timestamp") }, db::ColumnInfo { is_nullable: false, extra: Some("generated always as (id + 1) stored".to_string()), ..test_column("generated_id", "int") }, db::ColumnInfo { is_nullable: false, extra: Some("identity(1,1)".to_string()), ..test_column("sequence_id", "bigint") }, db::ColumnInfo { is_nullable: false, extra: Some("computed".to_string()), ..test_column("computed_id", "int") }, db::ColumnInfo { is_nullable: false, ..test_column("row_version", "rowversion") }, ]; let col_names = vec!["id".to_string()]; assert!(required_unmapped_transfer_target_columns( &target_columns[..6], &col_names, &DatabaseType::Mysql, true ) .is_empty()); assert!(required_unmapped_transfer_target_columns(&target_columns, &col_names, &DatabaseType::SqlServer, true) .is_empty()); } #[test] fn transfer_target_column_validation_rejects_required_unmapped_columns() { let target_columns = vec![ test_column("id", "int"), db::ColumnInfo { is_nullable: false, ..test_column("required_code", "varchar(32)") }, ]; let col_names = vec!["id".to_string()]; assert_eq!( required_unmapped_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Mysql, true), vec!["required_code".to_string()] ); } #[test] fn dameng_identity_insert_wrapper_quotes_schema_and_table() { let sql = wrap_dameng_identity_insert_sql( "INSERT INTO \"SYSDBA\".\"USERS\" (\"ID\") VALUES\n(1);", "USERS", "SYSDBA", ); assert_eq!( sql, "SET IDENTITY_INSERT \"SYSDBA\".\"USERS\" ON;\nINSERT INTO \"SYSDBA\".\"USERS\" (\"ID\") VALUES\n(1);\nSET IDENTITY_INSERT \"SYSDBA\".\"USERS\" OFF;" ); } #[test] fn sqlserver_identity_insert_statement_quotes_schema_and_table() { assert_eq!( identity_insert_statement("inter_putaway", "dbo", &DatabaseType::SqlServer, true), "SET IDENTITY_INSERT [dbo].[inter_putaway] ON" ); assert_eq!( identity_insert_statement("inter_putaway", "dbo", &DatabaseType::SqlServer, false), "SET IDENTITY_INSERT [dbo].[inter_putaway] OFF" ); } #[test] fn mysql_create_table_includes_column_comments() { let cols = vec![ db::ColumnInfo { comment: Some("用户ID".to_string()), is_primary_key: true, ..test_column("id", "int") }, db::ColumnInfo { comment: Some("用户姓名".to_string()), is_nullable: false, ..test_column("name", "VARCHAR(100)") }, db::ColumnInfo { comment: None, ..test_column("age", "int") }, ]; let ddl = generate_create_table_ddl(&cols, "users", "", "", &DatabaseType::Mysql, &DatabaseType::Mysql, None, None); assert!(ddl.contains("COMMENT '用户ID'")); assert!(ddl.contains("COMMENT '用户姓名'")); assert!(!ddl.contains("`age` INT COMMENT")); // no comment for age assert!(ddl.contains("`name` VARCHAR(100) NOT NULL COMMENT '用户姓名'")); assert!(ddl.contains("PRIMARY KEY (`id`)")); } #[test] fn gaussdb_transfer_can_leave_safe_target_columns_unquoted() { let columns = vec![ db::ColumnInfo { is_primary_key: true, ..test_column("USER_ID", "int") }, test_column("CamelName", "varchar(32)"), test_column("select", "varchar(32)"), test_column("has space", "varchar(32)"), ]; let ddl = generate_create_table_ddl_with_column_quoting( &columns, "case_target", "dbx_test", "public", &DatabaseType::Gaussdb, &DatabaseType::Mysql, None, None, false, ); assert!(ddl.contains("USER_ID INTEGER"), "ddl: {ddl}"); assert!(ddl.contains("CamelName VARCHAR(32)"), "ddl: {ddl}"); assert!(ddl.contains("\"select\" VARCHAR(32)"), "ddl: {ddl}"); assert!(ddl.contains("\"has space\" VARCHAR(32)"), "ddl: {ddl}"); assert!(ddl.contains("PRIMARY KEY (USER_ID)"), "ddl: {ddl}"); let quoted = generate_create_table_ddl_with_column_quoting( &columns, "case_target", "dbx_test", "public", &DatabaseType::Gaussdb, &DatabaseType::Mysql, None, None, true, ); assert!(quoted.contains("\"USER_ID\" INTEGER"), "ddl: {quoted}"); assert!(quoted.contains("\"CamelName\" VARCHAR(32)"), "ddl: {quoted}"); } #[test] fn gaussdb_transfer_writes_use_the_same_target_column_quoting_policy() { let columns = vec!["USER_ID".to_string(), "CamelName".to_string(), "select".to_string()]; let column_types = vec![Some("int".to_string()), Some("varchar".to_string()), Some("varchar".to_string())]; let rows = vec![vec![serde_json::json!(1), serde_json::json!("Alice"), serde_json::json!("ok")]]; let statements = generate_transfer_write_sql_batches_with_column_quoting( &TransferMode::Append, &columns, &column_types, &rows, "case_target", "public", &DatabaseType::Gaussdb, &[], None, false, false, false, ) .unwrap(); assert_eq!(statements.len(), 1); assert!( statements[0].starts_with("INSERT INTO \"public\".\"case_target\" (USER_ID, CamelName, \"select\") VALUES"), "sql: {}", statements[0] ); } #[test] fn oracle_to_mysql_create_table_ddl_strips_char_length_units() { // Issue #6479 end-to-end: Oracle columns reported with CHAR length // semantics must produce valid MySQL DDL (`VARCHAR(6)`, not // `VARCHAR(6 char)` which fails with ERROR 1064). let cols = vec![ test_column("ID", "VARCHAR2(6 CHAR)"), test_column("DWMC", "VARCHAR2(50 CHAR)"), test_column("JOB", "VARCHAR2(20 CHAR)"), test_column("FLAG", "CHAR(1 CHAR)"), ]; let ddl = generate_create_table_ddl(&cols, "USERS", "", "", &DatabaseType::Mysql, &DatabaseType::Oracle, None, None); assert!(ddl.contains("`ID` VARCHAR(6)"), "ddl: {ddl}"); assert!(ddl.contains("`DWMC` VARCHAR(50)"), "ddl: {ddl}"); assert!(ddl.contains("`JOB` VARCHAR(20)"), "ddl: {ddl}"); assert!(ddl.contains("`FLAG` CHAR(1)"), "ddl: {ddl}"); // A `char`/`CHAR` immediately before `)` would be a leaked Oracle unit // qualifier (`VARCHAR(6 char)`); a legit `CHAR(1)` does not match. assert!(!ddl.contains("char)"), "Oracle length unit leaked into DDL: {ddl}"); assert!(!ddl.contains("CHAR)"), "Oracle length unit leaked into DDL: {ddl}"); } #[test] fn dameng_create_table_omits_if_not_exists_without_changing_other_prefixes() { let cols = vec![test_column("id", "int")]; let dameng = generate_create_table_ddl( &cols, "users", "source", "SYSDBA", &DatabaseType::Dameng, &DatabaseType::Mysql, None, None, ); let mysql = generate_create_table_ddl( &cols, "users", "", "app", &DatabaseType::Mysql, &DatabaseType::Mysql, None, None, ); let postgres = generate_create_table_ddl( &cols, "users", "", "public", &DatabaseType::Postgres, &DatabaseType::Mysql, None, None, ); let sqlserver = generate_create_table_ddl( &cols, "users", "", "dbo", &DatabaseType::SqlServer, &DatabaseType::Mysql, None, None, ); assert!(dameng.starts_with("CREATE TABLE \"SYSDBA\".\"users\" ("), "ddl: {dameng}"); assert!(!dameng.contains("IF NOT EXISTS"), "ddl: {dameng}"); assert!(dameng.contains("\"id\" INTEGER"), "ddl: {dameng}"); assert!(mysql.starts_with("CREATE TABLE IF NOT EXISTS `users` ("), "ddl: {mysql}"); assert!(postgres.starts_with("CREATE TABLE IF NOT EXISTS \"public\".\"users\" ("), "ddl: {postgres}"); assert!( sqlserver.starts_with( "IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'users')\nCREATE TABLE [dbo].[users] (" ), "ddl: {sqlserver}" ); } #[test] fn postgres_create_table_preserves_defaults_identity_and_exact_types() { let cols = vec![ db::ColumnInfo { data_type: "integer".to_string(), column_default: Some("nextval('public.users_id_seq'::regclass)".to_string()), is_primary_key: true, is_nullable: false, ..test_column("id", "integer") }, db::ColumnInfo { data_type: "timestamp with time zone".to_string(), column_default: Some("now()".to_string()), is_nullable: false, ..test_column("created_at", "timestamp with time zone") }, db::ColumnInfo { data_type: "character varying(120)".to_string(), column_default: Some("'guest'::character varying".to_string()), ..test_column("name", "character varying(120)") }, ]; let ddl = generate_create_table_ddl( &cols, "users", "public", "public", &DatabaseType::Postgres, &DatabaseType::Postgres, None, None, ); assert!(ddl.contains("\"id\" integer GENERATED BY DEFAULT AS IDENTITY NOT NULL")); assert!(ddl.contains("\"created_at\" timestamp with time zone DEFAULT now() NOT NULL")); assert!(ddl.contains("\"name\" character varying(120) DEFAULT 'guest'::character varying")); assert!(ddl.contains("PRIMARY KEY (\"id\")")); } #[test] fn postgres_create_table_rewrites_schema_qualified_custom_types_and_defaults() { let cols = vec![db::ColumnInfo { data_type: "\"public\".\"user_status\"".to_string(), column_default: Some("'active'::public.user_status".to_string()), is_nullable: false, ..test_column("status", "\"public\".\"user_status\"") }]; let ddl = generate_create_table_ddl( &cols, "users", "public", "archive", &DatabaseType::Postgres, &DatabaseType::Postgres, None, None, ); assert!( ddl.contains("\"status\" \"archive\".\"user_status\" DEFAULT 'active'::\"archive\".user_status NOT NULL") ); } #[test] fn mysql_create_table_includes_table_comment() { let cols = vec![db::ColumnInfo { is_primary_key: true, ..test_column("id", "int") }]; let ddl = generate_create_table_ddl( &cols, "users", "", "", &DatabaseType::Mysql, &DatabaseType::Mysql, Some("用户表"), None, ); assert!(ddl.contains(") COMMENT='用户表'")); } #[test] fn mysql_text_pk_gets_key_prefix() { let cols = vec![db::ColumnInfo { data_type: "text".to_string(), is_primary_key: true, ..test_column("id", "text") }]; let ddl = generate_create_table_ddl(&cols, "logs", "", "", &DatabaseType::Mysql, &DatabaseType::Sqlite, None, None); assert!(ddl.contains("PRIMARY KEY (`id`(255))")); assert!(ddl.contains("`id` TEXT")); } #[test] fn mysql_int_pk_no_prefix() { let cols = vec![db::ColumnInfo { is_primary_key: true, ..test_column("id", "int") }]; let ddl = generate_create_table_ddl(&cols, "users", "", "", &DatabaseType::Mysql, &DatabaseType::Sqlite, None, None); assert!(ddl.contains("PRIMARY KEY (`id`)")); assert!(!ddl.contains("PRIMARY KEY (`id`(255))")); } #[test] fn postgres_comment_ddl_generates_column_and_table_comments() { let cols = vec![ db::ColumnInfo { comment: Some("主键".to_string()), ..test_column("id", "int") }, db::ColumnInfo { comment: Some("名称".to_string()), ..test_column("name", "varchar(100)") }, ]; let stmts = generate_comment_ddl(&cols, "items", "public", &DatabaseType::Postgres, Some("项目表")); assert_eq!(stmts.len(), 3); assert!(stmts[0].contains("COMMENT ON TABLE \"public\".\"items\" IS '项目表'")); assert!(stmts[1].contains("COMMENT ON COLUMN \"public\".\"items\".\"id\" IS '主键'")); assert!(stmts[2].contains("COMMENT ON COLUMN \"public\".\"items\".\"name\" IS '名称'")); } #[test] fn kingbase_comment_ddl_generates_and_escapes_comments() { let cols = vec![ db::ColumnInfo { comment: Some("owner's id".to_string()), ..test_column("id", "int") }, db::ColumnInfo { comment: Some("display name".to_string()), ..test_column("name", "varchar(100)") }, ]; let stmts = generate_comment_ddl(&cols, "items", "public", &DatabaseType::Kingbase, Some("team's items")); assert_eq!( stmts, vec![ "COMMENT ON TABLE \"public\".\"items\" IS 'team''s items'".to_string(), "COMMENT ON COLUMN \"public\".\"items\".\"id\" IS 'owner''s id'".to_string(), "COMMENT ON COLUMN \"public\".\"items\".\"name\" IS 'display name'".to_string(), ] ); } #[test] fn kingbase_comment_ddl_skips_empty_comments() { let cols = vec![ db::ColumnInfo { comment: None, ..test_column("id", "int") }, db::ColumnInfo { comment: Some(" ".to_string()), ..test_column("name", "varchar(100)") }, ]; let stmts = generate_comment_ddl(&cols, "items", "public", &DatabaseType::Kingbase, Some(" ")); assert!(stmts.is_empty()); } #[test] fn postgres_transfer_ddl_splits_reused_multi_statement_table_ddl() { let ddl = "CREATE TABLE \"public\".\"items\" (\"id\" integer);\nCOMMENT ON TABLE \"public\".\"items\" IS 'items';"; let statements = transfer_ddl_statements(ddl, &DatabaseType::Postgres); assert_eq!( statements, vec![ "CREATE TABLE \"public\".\"items\" (\"id\" integer)".to_string(), "COMMENT ON TABLE \"public\".\"items\" IS 'items'".to_string(), ] ); } #[test] fn dameng_transfer_ddl_splits_reused_table_comments() { let ddl = "CREATE TABLE \"APP\".\"ITEMS\" (\n\ \"ID\" INTEGER,\n\ \"NOTE\" VARCHAR(100)\n\ );\n\ COMMENT ON TABLE \"APP\".\"ITEMS\" IS 'owner''s; items';\n\ COMMENT ON COLUMN \"APP\".\"ITEMS\".\"NOTE\" IS 'line; two';"; let statements = transfer_ddl_statements(ddl, &DatabaseType::Dameng); assert_eq!( statements, vec![ "CREATE TABLE \"APP\".\"ITEMS\" (\n\ \"ID\" INTEGER,\n\ \"NOTE\" VARCHAR(100)\n\ )" .to_string(), "COMMENT ON TABLE \"APP\".\"ITEMS\" IS 'owner''s; items'".to_string(), "COMMENT ON COLUMN \"APP\".\"ITEMS\".\"NOTE\" IS 'line; two'".to_string(), ] ); } #[test] fn dameng_transfer_ddl_preserves_plsql_blocks_and_single_statements() { let script = "BEGIN\n\ EXECUTE IMMEDIATE 'CREATE TABLE \"APP\".\"AUDIT\" (\"ID\" INTEGER)';\n\ END;\n\ /\n\ COMMENT ON TABLE \"APP\".\"AUDIT\" IS 'audit';"; let statements = transfer_ddl_statements(script, &DatabaseType::Dameng); assert_eq!(statements.len(), 2); assert!(statements[0].starts_with("BEGIN\n")); assert!(statements[0].contains("EXECUTE IMMEDIATE 'CREATE TABLE")); assert!(statements[0].ends_with("END;")); assert_eq!(statements[1], "COMMENT ON TABLE \"APP\".\"AUDIT\" IS 'audit'"); let single = "CREATE TABLE \"APP\".\"SINGLE_ITEM\" (\"ID\" INTEGER)"; assert_eq!(transfer_ddl_statements(single, &DatabaseType::Dameng), vec![single.to_string()]); } #[test] fn postgres_transfer_ddl_skips_reused_index_statements() { let ddl = "CREATE TABLE \"public\".\"items\" (\"id\" integer);\n\ CREATE INDEX \"items_lower_idx\" ON \"public\".\"items\" USING btree (\"lower(name)\");\n\ COMMENT ON INDEX \"public\".\"items_lower_idx\" IS 'lookup';"; let statements = transfer_ddl_statements(ddl, &DatabaseType::Postgres); assert_eq!(statements, vec!["CREATE TABLE \"public\".\"items\" (\"id\" integer)".to_string()]); } #[test] fn postgres_transfer_ddl_removes_inline_foreign_keys_from_reused_table_ddl() { let ddl = "CREATE TABLE \"public\".\"audit_logs\" (\n \"id\" integer,\n \"user_id\" integer,\n CONSTRAINT \"audit_logs_user_id_fkey\" FOREIGN KEY (\"user_id\") REFERENCES \"users\"(\"id\")\n);"; let statements = transfer_ddl_statements(ddl, &DatabaseType::Postgres); assert_eq!( statements, vec!["CREATE TABLE \"public\".\"audit_logs\" (\n \"id\" integer,\n \"user_id\" integer\n)".to_string()] ); } #[test] fn postgres_transfer_ddl_preserves_constraint_after_inline_foreign_key() { let ddl = "CREATE TABLE \"source_7931\".\"dbx_child\" (\n \"id\" integer NOT NULL,\n \"parent_id\" integer NOT NULL,\n \"status\" integer NOT NULL,\n CONSTRAINT \"dbx_child_parent_fk\" FOREIGN KEY (\"parent_id\") REFERENCES \"source_7931\".\"dbx_parent\"(\"id\"),\n CONSTRAINT \"dbx_child_status_check\" CHECK (status >= 0)\n);"; let statements = transfer_ddl_statements(ddl, &DatabaseType::Postgres); assert_eq!( statements, vec!["CREATE TABLE \"source_7931\".\"dbx_child\" (\n \"id\" integer NOT NULL,\n \"parent_id\" integer NOT NULL,\n \"status\" integer NOT NULL,\n CONSTRAINT \"dbx_child_status_check\" CHECK (status >= 0)\n)".to_string()] ); } #[test] fn postgres_transfer_ddl_removes_multiple_inline_foreign_keys_without_damaging_retained_items() { let ddl = "CREATE TABLE \"public\".\"assignments\" (\n \"id\" integer NOT NULL,\n \"owner_id\" integer NOT NULL,\n \"reviewer_id\" integer NOT NULL,\n CONSTRAINT \"assignments_owner_fk\" FOREIGN KEY (\"owner_id\") REFERENCES \"users\"(\"id\"),\n CONSTRAINT \"assignments_owner_check\" CHECK (owner_id > 0),\n CONSTRAINT \"assignments_reviewer_fk\" FOREIGN KEY (\"reviewer_id\") REFERENCES \"users\"(\"id\"),\n CONSTRAINT \"assignments_owner_reviewer_unique\" UNIQUE (\"owner_id\", \"reviewer_id\")\n);"; let statements = transfer_ddl_statements(ddl, &DatabaseType::Postgres); assert_eq!( statements, vec!["CREATE TABLE \"public\".\"assignments\" (\n \"id\" integer NOT NULL,\n \"owner_id\" integer NOT NULL,\n \"reviewer_id\" integer NOT NULL,\n CONSTRAINT \"assignments_owner_check\" CHECK (owner_id > 0),\n CONSTRAINT \"assignments_owner_reviewer_unique\" UNIQUE (\"owner_id\", \"reviewer_id\")\n)".to_string()] ); } #[test] fn transfer_create_table_result_treats_existing_table_as_preexisting() { assert!(!transfer_create_table_created( Err("ERROR: relation \"items\" already exists (SQLSTATE 42P07)".to_string()), "create" ) .unwrap()); assert!(!transfer_create_table_created(Err("错误: 关系 \"items\" 已经存在".to_string()), "create").unwrap()); assert!(transfer_create_table_created(Ok(()), "create").unwrap()); assert_eq!( transfer_create_table_created(Err("permission denied for schema public".to_string()), "create") .unwrap_err(), "create: permission denied for schema public" ); } #[test] fn non_postgres_transfer_ddl_keeps_statement_text_intact() { let ddl = "CREATE TABLE `items` (`id` int);\nALTER TABLE `items` COMMENT = 'items';"; assert_eq!(transfer_ddl_statements(ddl, &DatabaseType::Mysql), vec![ddl.to_string()]); } #[test] fn clickhouse_comment_ddl_uses_alter_table() { let cols = vec![db::ColumnInfo { comment: Some("日志消息".to_string()), ..test_column("message", "text") }]; let stmts = generate_comment_ddl(&cols, "logs", "", &DatabaseType::ClickHouse, None); assert_eq!(stmts.len(), 1); assert!(stmts[0].contains("ALTER TABLE `logs` COMMENT COLUMN `message` '日志消息'")); } #[test] fn pg_comment_ddl_skips_empty_comments() { let cols = vec![ db::ColumnInfo { comment: None, ..test_column("id", "int") }, db::ColumnInfo { comment: Some(" ".to_string()), ..test_column("name", "varchar(100)") }, ]; let stmts = generate_comment_ddl(&cols, "t", "", &DatabaseType::Postgres, None); assert!(stmts.is_empty()); } #[test] fn non_mysql_family_no_inline_comment() { let cols = vec![db::ColumnInfo { comment: Some("test".to_string()), ..test_column("col", "text") }]; // PostgreSQL target should NOT have inline COMMENT let ddl = generate_create_table_ddl(&cols, "t", "", "", &DatabaseType::Postgres, &DatabaseType::Postgres, None, None); assert!(!ddl.contains("COMMENT")); } #[test] fn clickhouse_create_table_with_pk_uses_order_by_pk() { let cols = vec![ db::ColumnInfo { is_primary_key: true, is_nullable: false, ..test_column("id", "UInt64") }, db::ColumnInfo { ..test_column("name", "String") }, ]; let ddl = generate_create_table_ddl( &cols, "logs", "", "", &DatabaseType::ClickHouse, &DatabaseType::ClickHouse, None, None, ); // Must include ENGINE with ORDER BY using the PK columns assert!(ddl.contains("ENGINE = MergeTree() ORDER BY (`id`)")); // Must NOT have a separate PRIMARY KEY clause (ORDER BY serves that role) assert!(!ddl.contains("PRIMARY KEY")); } #[test] fn clickhouse_create_table_without_pk_uses_order_by_tuple() { let cols = vec![db::ColumnInfo { ..test_column("message", "String") }]; let ddl = generate_create_table_ddl( &cols, "logs", "", "", &DatabaseType::ClickHouse, &DatabaseType::ClickHouse, None, None, ); assert!(ddl.contains("ENGINE = MergeTree() ORDER BY tuple()")); assert!(!ddl.contains("PRIMARY KEY")); } #[test] fn clickhouse_transfer_maps_fractional_timestamp_to_datetime64() { let cols = vec![db::ColumnInfo { numeric_scale: Some(6), ..test_column("created_at", "TIMESTAMP") }]; let ddl = generate_create_table_ddl( &cols, "events", "SYSDBA", "", &DatabaseType::ClickHouse, &DatabaseType::Dameng, None, None, ); assert!(ddl.contains("`created_at` DateTime64(6)"), "ddl: {ddl}"); } #[test] fn clickhouse_transfer_uses_datetime64_fallback_for_timestamp_types() { assert_eq!(map_column_type("datetime", &DatabaseType::Dameng, &DatabaseType::ClickHouse), "DateTime64(6)"); assert_eq!(map_column_type("timestamp", &DatabaseType::Dameng, &DatabaseType::ClickHouse), "DateTime64(6)"); } #[test] fn transfer_reuses_source_table_ddl_only_when_target_shape_matches() { assert!(!can_reuse_source_table_ddl(&DatabaseType::ClickHouse, &DatabaseType::ClickHouse, None, None, true,)); assert!(can_reuse_source_table_ddl(&DatabaseType::Postgres, &DatabaseType::Postgres, None, None, true,)); assert!(!can_reuse_source_table_ddl(&DatabaseType::Postgres, &DatabaseType::Postgres, None, None, false,)); assert!(can_reuse_source_table_ddl(&DatabaseType::Dameng, &DatabaseType::Dameng, None, None, true,)); } #[test] fn oceanbase_mysql_transfer_only_reuses_ddl_for_an_oceanbase_target() { assert!( !can_reuse_source_table_ddl(&DatabaseType::Mysql, &DatabaseType::Mysql, Some("oceanbase"), None, true,) ); assert!(can_reuse_source_table_ddl( &DatabaseType::Mysql, &DatabaseType::Mysql, Some("OceanBase"), Some("oceanbase"), true, )); assert!(can_reuse_source_table_ddl(&DatabaseType::Mysql, &DatabaseType::Mysql, None, None, true,)); } #[test] fn mysql_transfer_collation_recovery_only_reads_ddl_code() { let ddl = r#"CREATE TABLE `COLLATE utf8mb4_identifier_ci` ( `id` bigint NOT NULL, `note` varchar(255) COMMENT 'COLLATE utf8mb4_literal_ci', `name` varchar(64) COLLATE utf8mb4_0900_ai_ci, `legacy` varchar(64) collate = utf8mb4_unicode_ci ) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci /* COLLATE utf8mb4_comment_ci */"#; assert_eq!( mysql_ddl_collation_names(ddl), vec!["utf8mb4_0900_ai_ci".to_string(), "utf8mb4_unicode_ci".to_string()] ); } #[test] fn mysql_transfer_collation_recovery_removes_only_unsupported_clauses() { let ddl = r#"CREATE TABLE `items` ( `name` varchar(64) COLLATE utf8mb4_0900_ai_ci COMMENT 'COLLATE utf8mb4_0900_ai_ci', `legacy` varchar(64) COLLATE utf8mb4_unicode_ci ) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='items'"#; let supported = HashSet::from(["utf8mb4_unicode_ci".to_string()]); let rewritten = remove_unsupported_mysql_collations(ddl, &supported); assert!(!rewritten.contains("varchar(64) COLLATE utf8mb4_0900_ai_ci")); assert!(!rewritten.contains("utf8mb4 COLLATE=utf8mb4_0900_ai_ci")); assert!(rewritten.contains("COLLATE utf8mb4_unicode_ci")); assert!(rewritten.contains("COMMENT 'COLLATE utf8mb4_0900_ai_ci'")); assert!(rewritten.contains("DEFAULT CHARSET=utf8mb4")); assert!(rewritten.contains("COMMENT='items'")); } #[test] fn mysql_transfer_collation_recovery_has_a_narrow_error_gate() { let ddl = "CREATE TABLE `items` (`name` varchar(64) COLLATE utf8mb4_0900_ai_ci)"; assert_eq!( mysql_collations_for_transfer_ddl_recovery( ddl, "ERROR 1273 (HY000): Unknown collation: 'utf8mb4_0900_ai_ci'", &DatabaseType::Mysql, true, ), Some(vec!["utf8mb4_0900_ai_ci".to_string()]) ); assert_eq!( mysql_collations_for_transfer_ddl_recovery( ddl, "ERROR 1064 (42000): syntax error", &DatabaseType::Mysql, true, ), None ); assert_eq!( mysql_collations_for_transfer_ddl_recovery( ddl, "ERROR 1273 (HY000): Unknown collation", &DatabaseType::Mysql, false, ), None ); assert_eq!( mysql_collations_for_transfer_ddl_recovery( ddl, "ERROR 1273 (HY000): Unknown collation", &DatabaseType::Postgres, true, ), None ); } #[test] fn detects_oceanbase_mysql_table_options_outside_literals_and_comments() { let ddl = r#"CREATE TABLE `items` ( `id` bigint NOT NULL AUTO_INCREMENT, KEY `idx_name` (`name`) BLOCK_SIZE 16384 LOCAL ) AUTO_INCREMENT = 42 AUTO_INCREMENT_MODE = 'ORDER' DEFAULT CHARSET = utf8mb4 REPLICA_NUM = 1 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 0"#; assert!(contains_oceanbase_mysql_table_options(ddl)); let portable = r#"CREATE TABLE `items` ( `id` bigint NOT NULL AUTO_INCREMENT, `note` varchar(255) COMMENT 'AUTO_INCREMENT_MODE and PCTFREE', PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='USE_BLOOM_FILTER'"#; assert!(!contains_oceanbase_mysql_table_options(portable)); } #[test] fn postgres_transfer_reused_table_ddl_rewrites_target_schema() { let ddl = "CREATE TABLE \"src\".\"items\" (\"id\" integer);\nCOMMENT ON COLUMN \"src\".\"items\".\"id\" IS 'id';"; let rewritten = rewrite_transfer_source_table_ddl(ddl, "src", "dst", &DatabaseType::Postgres, &DatabaseType::Postgres); assert!(rewritten.contains("CREATE TABLE \"dst\".\"items\"")); assert!(rewritten.contains("COMMENT ON COLUMN \"dst\".\"items\".\"id\"")); assert!(!rewritten.contains("\"src\".\"items\"")); } #[test] fn dameng_transfer_reused_table_ddl_rewrites_schema_and_strips_storage() { let ddl = concat!( "CREATE TABLE \"SRC\".\"items\" (\n", "\"ID\" BIGINT IDENTITY(1, 1) NOT NULL,\n", "\"NOTE\" VARCHAR(100) DEFAULT '\"SRC\".literal',\n", "NOT CLUSTER PRIMARY KEY(\"ID\")) ", "STORAGE(ON \"SOURCE_TS\", CLUSTERBTR);\n", "COMMENT ON TABLE \"SRC\".\"items\" IS 'keep STORAGE(ON literal_ts) and \"SRC\".comment';\n", "-- keep STORAGE(ON line_comment_ts) and \"SRC\".line_comment\n", "/* keep STORAGE(ON block_comment_ts) and \"SRC\".block_comment */\n", "ALTER TABLE \"SRC\".\"items\" ADD \"value\" INTEGER;", ); let rewritten = rewrite_transfer_source_table_ddl(ddl, "SRC", "DST", &DatabaseType::Dameng, &DatabaseType::Dameng); assert!(rewritten.contains("CREATE TABLE \"DST\".\"items\"")); assert!(rewritten.contains("COMMENT ON TABLE \"DST\".\"items\"")); assert!(rewritten.contains("ALTER TABLE \"DST\".\"items\"")); assert!(!rewritten.contains("STORAGE(ON \"SOURCE_TS\", CLUSTERBTR)")); assert!(rewritten.contains("\"ID\" BIGINT IDENTITY(1, 1) NOT NULL")); assert!(rewritten.contains("NOT CLUSTER PRIMARY KEY(\"ID\")")); assert!(rewritten.contains("'\"SRC\".literal'")); assert!(rewritten.contains("'keep STORAGE(ON literal_ts) and \"SRC\".comment'")); assert!(rewritten.contains("-- keep STORAGE(ON line_comment_ts) and \"SRC\".line_comment")); assert!(rewritten.contains("/* keep STORAGE(ON block_comment_ts) and \"SRC\".block_comment */")); let storage_portable_ddl = strip_dameng_storage_clauses(ddl); assert_eq!( rewrite_transfer_source_table_ddl(ddl, "SRC", "SRC", &DatabaseType::Dameng, &DatabaseType::Dameng), storage_portable_ddl ); assert_eq!( rewrite_transfer_source_table_ddl(ddl, "", "DST", &DatabaseType::Dameng, &DatabaseType::Dameng), storage_portable_ddl ); } #[test] fn hive_create_table_uses_hive_friendly_columns() { let cols = vec![ db::ColumnInfo { is_primary_key: true, is_nullable: false, ..test_column("id", "bigint") }, db::ColumnInfo { is_nullable: false, ..test_column("payload", "json") }, ]; let ddl = generate_create_table_ddl( &cols, "events", "public", "warehouse", &DatabaseType::Hive, &DatabaseType::Postgres, None, None, ); assert!(ddl.contains("CREATE TABLE IF NOT EXISTS `warehouse`.`events`")); assert!(ddl.contains("`id` BIGINT")); assert!(ddl.contains("`payload` STRING")); assert!(!ddl.contains("PRIMARY KEY")); assert!(!ddl.contains("NOT NULL")); } #[test] fn hive_transfer_uses_backticks_and_hive_type_mapping() { assert_eq!(quote_identifier("user`events", &DatabaseType::Hive), "`user``events`"); assert_eq!(map_column_type("jsonb", &DatabaseType::Postgres, &DatabaseType::Hive), "STRING"); assert_eq!( map_column_type("timestamp with time zone", &DatabaseType::Postgres, &DatabaseType::Hive), "TIMESTAMP" ); } #[test] fn kyuubi_transfer_uses_spark_sql_compatible_ddl_and_batches() { let cols = vec![ db::ColumnInfo { is_primary_key: true, is_nullable: false, ..test_column("id", "bigint") }, db::ColumnInfo { is_nullable: false, ..test_column("payload", "jsonb") }, ]; let ddl = generate_create_table_ddl( &cols, "events", "public", "warehouse", &DatabaseType::Kyuubi, &DatabaseType::Postgres, None, None, ); assert!(ddl.contains("CREATE TABLE IF NOT EXISTS `warehouse`.`events`")); assert!(ddl.contains("`id` BIGINT")); assert!(ddl.contains("`payload` STRING")); assert!(!ddl.contains("PRIMARY KEY")); assert!(!ddl.contains("NOT NULL")); assert_eq!(quote_identifier("user`events", &DatabaseType::Kyuubi), "`user``events`"); assert_eq!(max_transfer_write_rows(&DatabaseType::Kyuubi, &TransferMode::Append), 500); assert_eq!(max_transfer_write_rows(&DatabaseType::Kyuubi, &TransferMode::Upsert), 500); } #[test] fn impala_transfer_uses_impala_compatible_ddl_and_batches() { let cols = vec![ db::ColumnInfo { is_primary_key: true, is_nullable: false, ..test_column("id", "bigint") }, db::ColumnInfo { is_nullable: false, ..test_column("payload", "jsonb") }, db::ColumnInfo { is_nullable: true, ..test_column("binary_payload", "bytea") }, ]; let ddl = generate_create_table_ddl( &cols, "events", "public", "warehouse", &DatabaseType::Impala, &DatabaseType::Postgres, None, None, ); assert!(ddl.contains("CREATE TABLE IF NOT EXISTS `warehouse`.`events`")); assert!(ddl.contains("`id` BIGINT")); assert!(ddl.contains("`payload` STRING")); assert!(ddl.contains("`binary_payload` STRING")); assert!(!ddl.contains("PRIMARY KEY")); assert!(!ddl.contains("NOT NULL")); assert_eq!(map_column_type("varbinary(255)", &DatabaseType::Mysql, &DatabaseType::Impala), "STRING"); assert_eq!(map_column_type("bytea", &DatabaseType::Postgres, &DatabaseType::Impala), "STRING"); assert_eq!(map_column_type("binary", &DatabaseType::Mysql, &DatabaseType::Hive), "BINARY"); let statements = generate_transfer_write_sql_batches( &TransferMode::Append, &[String::from("id"), String::from("binary_payload")], &[Some(String::from("bigint")), Some(String::from("bytea"))], &[vec![json!(1), json!("0x00ff")]], "events", "warehouse", &DatabaseType::Impala, &[], None, false, false, ) .unwrap(); assert_eq!(statements, vec!["INSERT INTO `warehouse`.`events` (`id`, `binary_payload`) VALUES\n(1, '0x00ff')"]); assert_eq!(max_transfer_write_rows(&DatabaseType::Impala, &TransferMode::Append), 500); assert_eq!(max_transfer_write_rows(&DatabaseType::Impala, &TransferMode::Upsert), 500); } #[test] fn mongo_transfer_document_fields_preserve_first_seen_order() { let documents = vec![json!({"b": 1}), json!({"a": 2, "c": 3}), json!({"b": 4, "d": 5})]; assert_eq!(mongo_transfer_document_fields(&documents), vec!["b", "a", "c", "d"]); } #[test] fn mongo_transfer_rows_fill_missing_fields_with_null() { let rows = mongo_documents_to_rows( &[json!({"id": 1, "name": "Ada"}), json!({"id": 2})], &[String::from("id"), String::from("name")], ); assert_eq!(rows, vec![vec![json!(1), json!("Ada")], vec![json!(2), serde_json::Value::Null]]); } #[test] fn mongo_transfer_uses_extended_json_to_preserve_bson_dates() { let result = MongoDocumentResult { documents: vec![json!({"gmtModified": "ISODate(\"2026-08-31T10:14:01.105Z\")"})], raw_documents: None, extended_documents: Some(vec![json!({ "gmtModified": {"$date": {"$numberLong": "1788161641105"}} })]), total: 1, total_is_exact: true, next_cursor: None, }; let documents = mongo_documents_for_transfer(result, true); assert_eq!(documents[0]["gmtModified"]["$date"]["$numberLong"], json!("1788161641105")); } #[test] fn mongo_transfer_falls_back_to_browser_documents_without_extended_json() { let result = MongoDocumentResult { documents: vec![json!({"value": "text"})], raw_documents: None, extended_documents: None, total: 1, total_is_exact: true, next_cursor: None, }; assert_eq!(mongo_documents_for_transfer(result, true), vec![json!({"value": "text"})]); } #[test] fn mongo_transfer_keeps_browser_documents_for_sql_targets() { let result = MongoDocumentResult { documents: vec![json!({"gmtModified": "ISODate(\"2026-08-31T10:14:01.105Z\")"})], raw_documents: None, extended_documents: Some(vec![json!({ "gmtModified": {"$date": {"$numberLong": "1788161641105"}} })]), total: 1, total_is_exact: true, next_cursor: None, }; assert_eq!( mongo_documents_for_transfer(result, false), vec![json!({"gmtModified": "ISODate(\"2026-08-31T10:14:01.105Z\")"})] ); } #[test] fn mongo_transfer_falls_back_when_extended_json_row_count_differs() { let result = MongoDocumentResult { documents: vec![json!({"id": 1}), json!({"id": 2})], raw_documents: None, extended_documents: Some(vec![json!({"id": {"$numberInt": "1"}})]), total: 2, total_is_exact: true, next_cursor: None, }; assert_eq!(mongo_documents_for_transfer(result, true), vec![json!({"id": 1}), json!({"id": 2})]); } #[test] fn sql_rows_to_mongo_documents_maps_columns_to_fields() { let documents = sql_rows_to_mongo_documents( &[String::from("id"), String::from("name"), String::from("active")], &[vec![json!(1), json!("Ada")], vec![json!(2), json!("Grace"), json!(true)]], ); assert_eq!( documents, vec![json!({"id": 1, "name": "Ada", "active": null}), json!({"id": 2, "name": "Grace", "active": true})] ); } #[test] fn postgres_pagination_uses_stable_primary_key_order() { let sql = pagination_sql_with_order( &[String::from("id"), String::from("name")], "users", "public", &DatabaseType::Postgres, 200, 100, &[String::from("id")], None, ); assert_eq!(sql, "SELECT \"id\", \"name\" FROM \"public\".\"users\" ORDER BY \"id\" LIMIT 100 OFFSET 200"); } #[test] fn impala_transfer_without_primary_key_uses_one_cursor_query() { let sql = transfer_cursor_sql( &[String::from("id"), String::from("name")], "events", "analytics", &DatabaseType::Impala, None, ); assert_eq!(sql, "SELECT `id`, `name` FROM `analytics`.`events`"); assert!(!sql.contains("ORDER BY")); assert!(!sql.contains("LIMIT")); assert!(!sql.contains("OFFSET")); } #[test] fn impala_transfer_preserves_explicit_primary_key_order() { let sql = pagination_sql_with_order( &[String::from("id"), String::from("name")], "events", "analytics", &DatabaseType::Impala, 1000, 1000, &[String::from("id")], None, ); assert_eq!(sql, "SELECT `id`, `name` FROM `analytics`.`events` ORDER BY `id` LIMIT 1000 OFFSET 1000"); } #[test] fn doris_unique_key_columns_drive_transfer_pagination_order() { let columns = vec![db::ColumnInfo { is_unique: true, ..test_column("id", "int") }, test_column("payload", "varchar(64)")]; let key_columns = transfer_key_columns(&columns, &DatabaseType::Doris); assert_eq!(key_columns, vec![String::from("id")]); assert_eq!( pagination_sql_with_order( &[String::from("id"), String::from("payload")], "events", "analytics", &DatabaseType::Doris, 1000, 1000, &key_columns, None, ), "SELECT `id`, `payload` FROM `analytics`.`events` ORDER BY `id` LIMIT 1000 OFFSET 1000" ); } #[test] fn mysql_unique_columns_do_not_become_transfer_keys() { let columns = vec![db::ColumnInfo { is_unique: true, ..test_column("email", "varchar(255)") }]; assert!(transfer_key_columns(&columns, &DatabaseType::Mysql).is_empty()); } #[test] fn questdb_pagination_uses_stable_primary_key_order() { let sql = pagination_sql_with_order( &[String::from("id"), String::from("name")], "users", "public", &DatabaseType::Questdb, 200, 100, &[String::from("id")], None, ); assert_eq!(sql, "SELECT `id`, `name` FROM `users` ORDER BY `id` LIMIT 200, 300"); } #[test] fn informix_pagination_uses_first_and_optional_skip() { let columns = [String::from("id"), String::from("name")]; assert_eq!( pagination_sql(&columns, "users", "app", &DatabaseType::Informix, 0, 100), "SELECT FIRST 100 \"id\", \"name\" FROM \"app\".\"users\"" ); assert_eq!( pagination_sql(&columns, "users", "app", &DatabaseType::Informix, 200, 100), "SELECT SKIP 200 FIRST 100 \"id\", \"name\" FROM \"app\".\"users\"" ); } #[test] fn informix_ordered_pagination_uses_first_and_optional_skip() { let columns = [String::from("id"), String::from("name")]; let order = [String::from("id")]; assert_eq!( pagination_sql_with_order(&columns, "users", "app", &DatabaseType::Informix, 0, 100, &order, None), "SELECT FIRST 100 \"id\", \"name\" FROM \"app\".\"users\" ORDER BY id" ); assert_eq!( pagination_sql_with_order(&columns, "users", "app", &DatabaseType::Informix, 200, 100, &order, None), "SELECT SKIP 200 FIRST 100 \"id\", \"name\" FROM \"app\".\"users\" ORDER BY id" ); } #[test] fn informix_filtered_pagination_preserves_filter_and_order() { let columns = [String::from("id"), String::from("status")]; let default_order = [String::from("id")]; assert_eq!( pagination_sql_with_filter_order( &columns, "users", "app", &DatabaseType::Informix, 200, 100, Some("WHERE status = 'active'"), Some("id DESC"), &default_order, ), "SELECT SKIP 200 FIRST 100 id, status FROM \"app\".\"users\" WHERE (status = 'active') ORDER BY id DESC" ); } #[test] fn informix_keyset_pagination_uses_first() { assert_eq!( keyset_pagination_sql( &[String::from("id"), String::from("name")], "users", "app", &DatabaseType::Informix, &[String::from("id")], &[json!(25)], 100, ), "SELECT FIRST 100 id, name FROM \"app\".\"users\" WHERE id > 25 ORDER BY id ASC" ); } #[test] fn dameng_export_pagination_uses_offset_fetch() { let sql = pagination_sql( &[String::from("id"), String::from("name")], "users", "SYSDBA", &DatabaseType::Dameng, 500, 100, ); assert_eq!( sql, "SELECT \"id\", \"name\" FROM \"SYSDBA\".\"users\" ORDER BY (SELECT NULL) OFFSET 500 ROWS FETCH NEXT 100 ROWS ONLY" ); assert!(!sql.contains(" LIMIT ")); } #[test] fn dameng_ordered_pagination_uses_offset_fetch() { let sql = pagination_sql_with_order( &[String::from("id"), String::from("name")], "users", "SYSDBA", &DatabaseType::Dameng, 200, 100, &[String::from("id")], None, ); assert_eq!( sql, "SELECT \"id\", \"name\" FROM \"SYSDBA\".\"users\" ORDER BY \"id\" OFFSET 200 ROWS FETCH NEXT 100 ROWS ONLY" ); assert!(!sql.contains(" LIMIT ")); } #[test] fn sqlserver_export_pagination_uses_row_number_subquery() { let sql = pagination_sql( &[String::from("id"), String::from("name")], "users", "dbo", &DatabaseType::SqlServer, 500, 100, ); assert_eq!( sql, "SELECT [id], [name] FROM (SELECT [id], [name], ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) AS __dbx_row_num FROM [dbo].[users]) AS __dbx_page WHERE __dbx_row_num > 500 AND __dbx_row_num <= 600" ); assert!(!sql.contains("OFFSET")); assert!(!sql.contains(" FETCH ")); } #[test] fn sqlserver_ordered_pagination_uses_row_number_subquery() { let sql = pagination_sql_with_order( &[String::from("id"), String::from("name")], "users", "dbo", &DatabaseType::SqlServer, 200, 100, &[String::from("id")], None, ); assert!(sql.contains("ROW_NUMBER() OVER (ORDER BY [id]) AS __dbx_row_num")); assert!(sql.contains("WHERE __dbx_row_num > 200 AND __dbx_row_num <= 300")); assert!(!sql.contains("OFFSET")); assert!(!sql.contains(" FETCH ")); } #[test] fn sqlserver_filtered_pagination_preserves_filter_in_subquery() { let sql = pagination_sql_with_filter_order( &[String::from("id"), String::from("status")], "users", "dbo", &DatabaseType::SqlServer, 10_000, 2_000, Some("WHERE status = 'active'"), Some("[id] DESC"), &[String::from("id")], ); assert!(sql.contains("ROW_NUMBER() OVER (ORDER BY [id] DESC) AS __dbx_row_num")); assert!(sql.contains("FROM [dbo].[users] WHERE (status = 'active')")); assert!(sql.contains("WHERE __dbx_row_num > 10000 AND __dbx_row_num <= 12000")); assert!(!sql.contains("OFFSET")); assert!(!sql.contains(" FETCH ")); } #[test] fn filtered_pagination_preserves_where_and_order() { let sql = pagination_sql_with_filter_order( &[String::from("id"), String::from("status")], "users", "public", &DatabaseType::SapHana, 10_000, 2_000, Some("WHERE status = 'active'"), Some("\"id\" DESC"), &[String::from("id")], ); assert_eq!( sql, "SELECT \"id\", \"status\" FROM \"public\".\"users\" WHERE (status = 'active') ORDER BY \"id\" DESC LIMIT 2000 OFFSET 10000" ); } #[test] fn dameng_filtered_pagination_preserves_where_and_order() { let sql = pagination_sql_with_filter_order( &[String::from("id"), String::from("status")], "users", "SYSDBA", &DatabaseType::Dameng, 10_000, 2_000, Some("WHERE status = 'active'"), Some("\"id\" DESC"), &[String::from("id")], ); assert_eq!( sql, "SELECT \"id\", \"status\" FROM \"SYSDBA\".\"users\" WHERE (status = 'active') ORDER BY \"id\" DESC OFFSET 10000 ROWS FETCH NEXT 2000 ROWS ONLY" ); assert!(!sql.contains(" LIMIT ")); } #[test] fn oracle_filtered_pagination_uses_rownum_for_legacy_compatibility() { let sql = pagination_sql_with_filter_order( &[String::from("id"), String::from("status")], "users", "APP", &DatabaseType::Oracle, 10_000, 2_000, Some("WHERE status = 'active'"), Some("\"id\" DESC"), &[String::from("id")], ); assert_eq!( sql, "SELECT \"id\", \"status\" FROM (SELECT dbx_inner.*, ROWNUM AS \"__dbx_row_num\" FROM (SELECT \"id\", \"status\" FROM \"APP\".\"users\" WHERE (status = 'active') ORDER BY \"id\" DESC) dbx_inner WHERE ROWNUM <= 12000) WHERE \"__dbx_row_num\" > 10000" ); assert!(!sql.contains("OFFSET")); assert!(!sql.contains("FETCH NEXT")); } #[test] fn filtered_count_preserves_where() { let sql = count_sql_with_where("users", "public", &DatabaseType::SapHana, Some("WHERE status = 'active'"), None); assert_eq!(sql, "SELECT COUNT(*) FROM \"public\".\"users\" WHERE (status = 'active')"); } #[test] fn sqlserver_keyset_pagination_uses_top() { let sql = keyset_pagination_sql( &[String::from("id"), String::from("name")], "users", "dbo", &DatabaseType::SqlServer, &[String::from("id")], &[], 100, ); assert_eq!(sql, "SELECT TOP (100) [id], [name] FROM [dbo].[users] ORDER BY [id] ASC"); } #[test] fn dameng_keyset_pagination_includes_offset_fetch() { let sql = keyset_pagination_sql( &[String::from("id"), String::from("name")], "users", "SYSDBA", &DatabaseType::Dameng, &[String::from("id")], &[json!(25)], 100, ); assert_eq!( sql, "SELECT \"id\", \"name\" FROM \"SYSDBA\".\"users\" WHERE \"id\" > 25 ORDER BY \"id\" ASC OFFSET 0 ROWS FETCH NEXT 100 ROWS ONLY" ); assert!(!sql.contains(" LIMIT ")); } #[test] fn oracle_keyset_pagination_uses_rownum_for_legacy_compatibility() { let sql = keyset_pagination_sql( &[String::from("id"), String::from("name")], "users", "APP", &DatabaseType::Oracle, &[String::from("id")], &[json!(25)], 100, ); assert_eq!( sql, "SELECT \"id\", \"name\" FROM (SELECT \"id\", \"name\" FROM \"APP\".\"users\" WHERE \"id\" > 25 ORDER BY \"id\" ASC) WHERE ROWNUM <= 100" ); assert!(!sql.contains("OFFSET")); assert!(!sql.contains("FETCH NEXT")); } #[test] fn oceanbase_oracle_pagination_uses_rownum_for_all_transfer_paths() { let columns = [String::from("id"), String::from("name")]; let order = [String::from("id")]; for database_type in [DatabaseType::Oracle, DatabaseType::OceanbaseOracle] { let sql = pagination_sql(&columns, "users", "APP", &database_type, 100, 50); assert!(sql.contains("ROWNUM"), "database_type={database_type:?}, sql={sql}"); assert!(!sql.contains(" LIMIT "), "database_type={database_type:?}, sql={sql}"); let sql = pagination_sql_with_order(&columns, "users", "APP", &database_type, 100, 50, &order, None); assert!(sql.contains("ROWNUM"), "database_type={database_type:?}, sql={sql}"); assert!(sql.contains("ORDER BY \"id\""), "database_type={database_type:?}, sql={sql}"); assert!(!sql.contains(" LIMIT "), "database_type={database_type:?}, sql={sql}"); let sql = pagination_sql_with_filter_order( &columns, "users", "APP", &database_type, 100, 50, Some("WHERE status = 'active'"), Some("id DESC"), &order, ); assert!(sql.contains("ROWNUM"), "database_type={database_type:?}, sql={sql}"); assert!(sql.contains("WHERE (status = 'active')"), "database_type={database_type:?}, sql={sql}"); assert!(!sql.contains(" LIMIT "), "database_type={database_type:?}, sql={sql}"); let sql = keyset_pagination_sql( &columns, "users", "APP", &database_type, &[String::from("id")], &[json!(25)], 50, ); assert!(sql.contains("ROWNUM"), "database_type={database_type:?}, sql={sql}"); assert!(sql.contains("WHERE \"id\" > 25"), "database_type={database_type:?}, sql={sql}"); assert!(!sql.contains(" LIMIT "), "database_type={database_type:?}, sql={sql}"); } } #[test] fn composite_keyset_pagination_uses_portable_lexicographic_predicate() { let sql = keyset_pagination_sql( &[String::from("tenant_id"), String::from("id"), String::from("name")], "users", "dbo", &DatabaseType::SqlServer, &[String::from("tenant_id"), String::from("id")], &[json!(10), json!(25)], 100, ); assert_eq!( sql, "SELECT TOP (100) [tenant_id], [id], [name] FROM [dbo].[users] WHERE ([tenant_id] > 10 OR ([tenant_id] = 10 AND [id] > 25)) ORDER BY [tenant_id] ASC, [id] ASC" ); } #[test] fn postgres_generates_index_and_foreign_key_sql() { let indexes = vec![db::IndexInfo { name: "users_name_idx".to_string(), columns: vec!["lower(name)".to_string()], is_unique: false, is_primary: false, filter: Some("name IS NOT NULL".to_string()), index_type: Some("btree".to_string()), included_columns: Some(vec!["created_at".to_string()]), comment: Some("lookup index".to_string()), key_is_expression: Vec::new(), }]; let foreign_keys = vec![ db::ForeignKeyInfo { name: "orders_user_id_fkey".to_string(), column: "user_id".to_string(), ref_schema: None, ref_table: "users".to_string(), ref_column: "id".to_string(), on_update: None, on_delete: None, }, db::ForeignKeyInfo { name: "orders_user_id_fkey".to_string(), column: "tenant_id".to_string(), ref_schema: None, ref_table: "users".to_string(), ref_column: "tenant_id".to_string(), on_update: None, on_delete: None, }, ]; let index_sql = generate_postgres_index_ddl(&indexes, "users", "public"); let foreign_key_sql = generate_postgres_foreign_key_ddl(&foreign_keys, "orders", "public", "archive"); assert_eq!( index_sql, vec![ "CREATE INDEX IF NOT EXISTS \"users_name_idx\" ON \"public\".\"users\" USING btree (lower(name)) INCLUDE (\"created_at\") WHERE name IS NOT NULL".to_string(), "COMMENT ON INDEX \"public\".\"users_name_idx\" IS 'lookup index'".to_string(), ] ); assert_eq!( foreign_key_sql, vec![ "ALTER TABLE \"archive\".\"orders\" ADD CONSTRAINT \"orders_user_id_fkey\" FOREIGN KEY (\"user_id\", \"tenant_id\") REFERENCES \"archive\".\"users\" (\"id\", \"tenant_id\")".to_string() ] ); } #[test] fn postgres_sequence_sync_sql_uses_table_max_values() { let sql = generate_postgres_sequence_sync_sql( &[ db::ColumnInfo { name: "id".to_string(), data_type: "integer".to_string(), is_nullable: false, column_default: Some("nextval('public.users_id_seq'::regclass)".to_string()), is_primary_key: true, extra: None, comment: None, numeric_precision: None, numeric_scale: None, character_maximum_length: None, enum_values: None, ..Default::default() }, db::ColumnInfo { name: "identity_id".to_string(), data_type: "integer".to_string(), is_nullable: false, column_default: None, is_primary_key: false, extra: Some("generated by default as identity".to_string()), comment: None, numeric_precision: None, numeric_scale: None, character_maximum_length: None, enum_values: None, ..Default::default() }, db::ColumnInfo { name: "computed_id".to_string(), data_type: "integer".to_string(), is_nullable: false, column_default: None, is_primary_key: false, extra: Some("generated always as (identity_id + 1) stored".to_string()), comment: None, numeric_precision: None, numeric_scale: None, character_maximum_length: None, enum_values: None, ..Default::default() }, ], "users", "public", ); assert_eq!( sql, vec![ "SELECT setval(pg_get_serial_sequence('\"public\".\"users\"', 'id')::regclass, GREATEST(COALESCE(MAX(\"id\"::bigint), 0), 1), MAX(\"id\"::bigint) IS NOT NULL) FROM \"public\".\"users\"".to_string(), "SELECT setval(pg_get_serial_sequence('\"public\".\"users\"', 'identity_id')::regclass, GREATEST(COALESCE(MAX(\"identity_id\"::bigint), 0), 1), MAX(\"identity_id\"::bigint) IS NOT NULL) FROM \"public\".\"users\"".to_string() ] ); } #[test] fn postgres_sequence_sync_sql_casts_text_and_numeric_columns_to_bigint() { let sql = generate_postgres_sequence_sync_sql( &[ db::ColumnInfo { name: "text_id".to_string(), data_type: "text".to_string(), is_nullable: false, column_default: Some("nextval('public.dbx_text_id_seq'::regclass)::text".to_string()), is_primary_key: true, extra: None, comment: None, numeric_precision: None, numeric_scale: None, character_maximum_length: None, enum_values: None, ..Default::default() }, db::ColumnInfo { name: "numeric_id".to_string(), data_type: "numeric".to_string(), is_nullable: false, column_default: Some("nextval('public.dbx_numeric_id_seq'::regclass)".to_string()), is_primary_key: false, extra: None, comment: None, numeric_precision: None, numeric_scale: None, character_maximum_length: None, enum_values: None, ..Default::default() }, db::ColumnInfo { name: "plain_text".to_string(), data_type: "text".to_string(), is_nullable: true, column_default: None, is_primary_key: false, extra: None, comment: None, numeric_precision: None, numeric_scale: None, character_maximum_length: None, enum_values: None, ..Default::default() }, ], "seq_test", "public", ); assert_eq!( sql, vec![ "SELECT setval(pg_get_serial_sequence('\"public\".\"seq_test\"', 'text_id')::regclass, GREATEST(COALESCE(MAX(\"text_id\"::bigint), 0), 1), MAX(\"text_id\"::bigint) IS NOT NULL) FROM \"public\".\"seq_test\"".to_string(), "SELECT setval(pg_get_serial_sequence('\"public\".\"seq_test\"', 'numeric_id')::regclass, GREATEST(COALESCE(MAX(\"numeric_id\"::bigint), 0), 1), MAX(\"numeric_id\"::bigint) IS NOT NULL) FROM \"public\".\"seq_test\"".to_string(), ] ); } #[test] fn postgres_transfer_owned_sequence_ddl_uses_precreate_and_post_bind_steps() { let sequence = PostgresOwnedSequence { name: "it_quick_entry_id_seq".to_string(), owner_table: "it_quick_entry".to_string(), owner_column: "id".to_string(), }; let create_sql = format!("CREATE SEQUENCE {}", postgres_sequence_qualified_name("public", &sequence.name)); let owner_sql = format!( "ALTER SEQUENCE {} OWNED BY {}.{}", postgres_sequence_qualified_name("public", &sequence.name), qualified_table(&sequence.owner_table, "public", &DatabaseType::Postgres, None), quote_identifier(&sequence.owner_column, &DatabaseType::Postgres) ); assert_eq!(create_sql, "CREATE SEQUENCE \"public\".\"it_quick_entry_id_seq\"".to_string()); assert_eq!( owner_sql, "ALTER SEQUENCE \"public\".\"it_quick_entry_id_seq\" OWNED BY \"public\".\"it_quick_entry\".\"id\"" .to_string() ); } #[test] fn postgres_owned_sequence_state_detects_conflicting_existing_sequence() { let source = PostgresOwnedSequence { name: "it_quick_entry_id_seq".to_string(), owner_table: "it_quick_entry".to_string(), owner_column: "id".to_string(), }; let conflicting = PostgresSequenceSnapshot { name: "it_quick_entry_id_seq".to_string(), owner_table: Some("other_table".to_string()), owner_column: Some("id".to_string()), }; let error = validate_existing_postgres_sequence(&source, Some(&conflicting), "archive").unwrap_err(); assert!(error.contains("\"archive\".\"it_quick_entry_id_seq\"")); assert!(error.contains("already exists with incompatible ownership")); } #[test] fn postgres_transfer_reused_table_ddl_preserves_serial_sequence_dependencies() { let columns = vec![ db::ColumnInfo { name: "id".to_string(), data_type: "integer".to_string(), is_nullable: false, column_default: Some("nextval('public.it_quick_entry_id_seq'::regclass)".to_string()), is_primary_key: true, extra: None, comment: None, numeric_precision: None, numeric_scale: None, character_maximum_length: None, enum_values: None, ..Default::default() }, db::ColumnInfo { name: "name".to_string(), data_type: "text".to_string(), is_nullable: false, column_default: None, is_primary_key: false, extra: None, comment: None, numeric_precision: None, numeric_scale: None, character_maximum_length: None, enum_values: None, ..Default::default() }, ]; let source_ddl = crate::schema::render_postgres_table_ddl("public", "it_quick_entry", &columns, &[], &[], None); let rewritten = rewrite_transfer_source_table_ddl( &source_ddl, "public", "archive", &DatabaseType::Postgres, &DatabaseType::Postgres, ); let sequence = PostgresOwnedSequence { name: "it_quick_entry_id_seq".to_string(), owner_table: "it_quick_entry".to_string(), owner_column: "id".to_string(), }; let create_sql = format!("CREATE SEQUENCE {}", postgres_sequence_qualified_name("archive", &sequence.name)); let owner_sql = format!( "ALTER SEQUENCE {} OWNED BY {}.{}", postgres_sequence_qualified_name("archive", &sequence.name), qualified_table(&sequence.owner_table, "archive", &DatabaseType::Postgres, None), quote_identifier(&sequence.owner_column, &DatabaseType::Postgres) ); let sequence_sync_sql = generate_postgres_sequence_sync_sql(&columns, "it_quick_entry", "archive"); assert!(source_ddl.starts_with("CREATE TABLE \"public\".\"it_quick_entry\"")); assert!(!source_ddl.contains("CREATE SEQUENCE")); assert!(rewritten.contains("CREATE TABLE \"archive\".\"it_quick_entry\"")); assert!(rewritten.contains("nextval('\"archive\".it_quick_entry_id_seq'::regclass)")); assert_eq!(create_sql, "CREATE SEQUENCE \"archive\".\"it_quick_entry_id_seq\"".to_string()); assert_eq!( owner_sql, "ALTER SEQUENCE \"archive\".\"it_quick_entry_id_seq\" OWNED BY \"archive\".\"it_quick_entry\".\"id\"" .to_string() ); assert_eq!( sequence_sync_sql, vec![ "SELECT setval(pg_get_serial_sequence('\"archive\".\"it_quick_entry\"', 'id')::regclass, GREATEST(COALESCE(MAX(\"id\"::bigint), 0), 1), MAX(\"id\"::bigint) IS NOT NULL) FROM \"archive\".\"it_quick_entry\"".to_string() ] ); } #[test] fn postgres_routine_schema_rewrite_targets_destination_schema() { let rewritten = rewrite_postgres_routine_schema( "CREATE OR REPLACE FUNCTION public.bump_counter(id integer)\nRETURNS integer\nLANGUAGE plpgsql\nAS $$ BEGIN INSERT INTO public.audit_logs(user_id) VALUES (id); RETURN id + 1; END; $$", "public", "archive", ) .unwrap(); assert!(rewritten.starts_with("CREATE OR REPLACE FUNCTION \"archive\".\"bump_counter\"(")); assert!(rewritten.contains("INSERT INTO \"archive\".audit_logs")); } #[test] fn postgres_trigger_schema_rewrite_targets_destination_table() { let rewritten = rewrite_postgres_trigger_table_schema( "CREATE TRIGGER bump BEFORE INSERT ON public.users FOR EACH ROW EXECUTE FUNCTION public.bump_counter()", "public", "users", "archive", ); assert!(rewritten.contains(" ON \"archive\".\"users\" ")); assert!(rewritten.contains("EXECUTE FUNCTION \"archive\".bump_counter()")); } #[test] fn postgres_extension_enum_and_domain_ddl_is_repeatable() { let extension_sql = generate_postgres_extension_ddl( &PostgresExtensionSource { extension_name: "pgcrypto".to_string() }, "archive", ); let enum_sql = generate_postgres_enum_ddl( &PostgresEnumSource { type_name: "status".to_string(), labels: vec!["pending".to_string(), "done".to_string()], }, "archive", ); let domain_sql = generate_postgres_domain_ddl( &PostgresDomainSource { domain_name: "email".to_string(), base_type: "text".to_string(), default_value: Some("'unknown@example.com'::text".to_string()), not_null: true, checks: vec!["CHECK ((VALUE ~* '^[^@]+@[^@]+$'::text))".to_string()], }, "archive", ); assert_eq!(extension_sql, "CREATE EXTENSION IF NOT EXISTS \"pgcrypto\" WITH SCHEMA \"archive\""); assert!(enum_sql.contains("DO $$ BEGIN IF NOT EXISTS")); assert!(enum_sql.contains("CREATE TYPE \"archive\".\"status\" AS ENUM ('pending', 'done')")); assert!(domain_sql.contains("CREATE DOMAIN \"archive\".\"email\" AS text DEFAULT 'unknown@example.com'::text NOT NULL CHECK ((VALUE ~* '^[^@]+@[^@]+$'::text))")); } #[test] fn postgres_materialized_view_ddls_drop_and_recreate_in_target_schema() { let ddls = generate_postgres_materialized_view_ddls( &PostgresMaterializedViewSource { view_name: "active_users".to_string(), source: "SELECT id, name FROM public.users WHERE active".to_string(), }, "archive", ); assert_eq!(ddls.len(), 2); assert_eq!(ddls[0], "DROP MATERIALIZED VIEW IF EXISTS \"archive\".\"active_users\""); assert_eq!( ddls[1], "CREATE MATERIALIZED VIEW \"archive\".\"active_users\" AS\nSELECT id, name FROM public.users WHERE active;" ); } #[test] fn mysql_insert_normalizes_rfc3339_datetime_strings() { let sql = generate_insert_typed( &[String::from("insurance_start_time")], &[Some(String::from("datetime"))], &[vec![json!("2026-05-12T00:00:00+00:00")]], "policies", "", &DatabaseType::Mysql, None, ); assert_eq!(sql, "INSERT INTO `policies` (`insurance_start_time`) VALUES\n('2026-05-12 00:00:00')"); } #[test] fn count_sql_uses_three_part_name_for_mysql_external_catalog() { let sql = count_sql("t1", "ads", &DatabaseType::Mysql, Some("hive_catalog")); assert_eq!(sql, "SELECT COUNT(*) FROM `hive_catalog`.`ads`.`t1`"); } #[test] fn count_sql_uses_three_part_name_for_starrocks_external_catalog() { let sql = count_sql("t1", "ads", &DatabaseType::StarRocks, Some("paimon")); assert_eq!(sql, "SELECT COUNT(*) FROM `paimon`.`ads`.`t1`"); } #[test] fn mysql_insert_uses_column_types_for_temporal_literals() { let sql = generate_insert_typed( &[String::from("dt"), String::from("raw_text"), String::from("d"), String::from("t")], &[ Some(String::from("datetime")), Some(String::from("varchar(64)")), Some(String::from("date")), Some(String::from("time")), ], &[vec![ json!("2026-05-12T00:00:00+00:00"), json!("2026-05-12T00:00:00+00:00"), json!("2026-05-12T00:00:00+00:00"), json!("2026-05-12T09:30:45+00:00"), ]], "policies", "", &DatabaseType::Mysql, None, ); assert_eq!( sql, "INSERT INTO `policies` (`dt`, `raw_text`, `d`, `t`) VALUES\n('2026-05-12 00:00:00', '2026-05-12T00:00:00+00:00', '2026-05-12', '09:30:45')" ); } #[test] fn oracle_insert_uses_date_literals_for_date_columns() { let sql = generate_insert_typed( &[String::from("id"), String::from("created_on"), String::from("created_at"), String::from("raw_text")], &[ Some(String::from("NUMBER")), Some(String::from("DATE")), Some(String::from("TIMESTAMP(6)")), Some(String::from("VARCHAR2(64)")), ], &[vec![ json!(1), json!("2022-08-25T09:58:43Z"), json!("2022-08-25T09:58:43Z"), json!("2022-08-25T09:58:43Z"), ]], "events", "APP", &DatabaseType::Oracle, None, ); assert_eq!( sql, "INSERT INTO \"APP\".\"events\" (\"id\", \"created_on\", \"created_at\", \"raw_text\") VALUES\n(1, TO_DATE('2022-08-25 09:58:43', 'YYYY-MM-DD HH24:MI:SS'), TO_TIMESTAMP('2022-08-25 09:58:43', 'YYYY-MM-DD HH24:MI:SS'), '2022-08-25T09:58:43Z')" ); assert_eq!( escape_value_typed(&json!("2022-08-25T00:00:00Z"), &DatabaseType::Oracle, Some("DATE")), "DATE '2022-08-25'" ); assert_eq!( escape_value_typed( &json!("2022-08-25T09:58:43.123456+08:00"), &DatabaseType::Oracle, Some("TIMESTAMP(6) WITH TIME ZONE") ), "TO_TIMESTAMP_TZ('2022-08-25 09:58:43.123456 +08:00', 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM')" ); } #[test] fn mysql_insert_formats_numeric_strings_from_numeric_columns_as_numeric_literals() { let sql = generate_insert_typed( &[ String::from("id"), String::from("amount"), String::from("quantity"), String::from("text_id"), String::from("bad_number"), String::from("missing"), ], &[ Some(String::from("bigint(20)")), Some(String::from("decimal(10,2)")), Some(String::from("int unsigned")), Some(String::from("varchar(64)")), Some(String::from("bigint(20)")), Some(String::from("bigint(20)")), ], &[vec![ json!("1234567890123"), json!("12.34"), json!("42"), json!("123"), json!("not-a-number"), serde_json::Value::Null, ]], "orders", "", &DatabaseType::Mysql, None, ); assert_eq!( sql, "INSERT INTO `orders` (`id`, `amount`, `quantity`, `text_id`, `bad_number`, `missing`) VALUES\n(1234567890123, 12.34, 42, '123', 'not-a-number', NULL)" ); } #[test] fn mysql_upsert_formats_numeric_strings_from_numeric_columns_as_numeric_literals() { let sql = generate_upsert_typed( &[String::from("id"), String::from("amount")], &[Some(String::from("bigint(20)")), Some(String::from("decimal(10,2)"))], &[vec![json!("1234567890123"), json!("12.34")]], "orders", "", &DatabaseType::Mysql, &[String::from("id")], None, ); assert_eq!( sql, "INSERT INTO `orders` (`id`, `amount`) VALUES\n(1234567890123, 12.34)\nON DUPLICATE KEY UPDATE `amount` = VALUES(`amount`)" ); } #[test] fn sqlserver_insert_prefixes_string_literals_as_unicode() { let sql = generate_insert_typed( &[String::from("name"), String::from("note")], &[Some(String::from("nvarchar(100)")), Some(String::from("varchar(100)"))], &[vec![json!("Tiếng Việt"), json!("O'Brien")]], "customers", "dbo", &DatabaseType::SqlServer, None, ); assert_eq!(sql, "INSERT INTO [dbo].[customers] ([name], [note]) VALUES\n(N'Tiếng Việt', N'O''Brien')"); } #[test] fn sqlserver_insert_preserves_backslashes_control_characters_quotes_and_unicode() { let sql = generate_insert_typed( &[String::from("escape_sequence"), String::from("line_break"), String::from("quote_and_unicode")], &[ Some(String::from("nvarchar(max)")), Some(String::from("nvarchar(max)")), Some(String::from("nvarchar(max)")), ], &[vec![json!(r#"\n"#), json!("line1\r\nline2"), json!("O'Brien / Tiếng Việt")]], "notes", "dbo", &DatabaseType::SqlServer, None, ); assert_eq!( sql, "INSERT INTO [dbo].[notes] ([escape_sequence], [line_break], [quote_and_unicode]) VALUES\n(N'\\n', N'line1\r\nline2', N'O''Brien / Tiếng Việt')" ); } #[test] fn sqlserver_insert_formats_datetime_literals_with_supported_precision() { let sql = generate_insert_typed( &[String::from("id"), String::from("date1"), String::from("date2"), String::from("note")], &[ Some(String::from("int")), Some(String::from("datetime")), Some(String::from("datetime2(7)")), Some(String::from("nvarchar(100)")), ], &[vec![ json!(1), json!("2026-06-29 10:11:12.896666666"), json!("2026-06-29T10:11:12.8966666Z"), json!("2026-06-29 10:11:12.896666666"), ]], "test", "dbo", &DatabaseType::SqlServer, None, ); assert_eq!( sql, "INSERT INTO [dbo].[test] ([id], [date1], [date2], [note]) VALUES\n(1, N'2026-06-29 10:11:12.897', N'2026-06-29 10:11:12.8966666', N'2026-06-29 10:11:12.896666666')" ); } #[test] fn sqlserver_insert_formats_bit_booleans_as_numeric_literals() { let sql = generate_insert_typed( &[String::from("enabled"), String::from("deleted")], &[Some(String::from("bit")), Some(String::from("BIT"))], &[vec![json!(true), json!(false)]], "flags", "dbo", &DatabaseType::SqlServer, None, ); assert_eq!(sql, "INSERT INTO [dbo].[flags] ([enabled], [deleted]) VALUES\n(1, 0)"); } #[test] fn sqlserver_insert_formats_prefixed_hex_for_varbinary_columns() { let sql = generate_insert_typed( &[String::from("payload"), String::from("note")], &[Some(String::from("varbinary(max)")), Some(String::from("nvarchar(64)"))], &[vec![json!("0x0001ABff"), json!("0x0001ABff")]], "files", "dbo", &DatabaseType::SqlServer, None, ); assert_eq!(sql, "INSERT INTO [dbo].[files] ([payload], [note]) VALUES\n(0x0001ABff, N'0x0001ABff')"); } #[test] fn sqlserver_insert_explicitly_converts_plain_text_for_varbinary_columns() { let sql = generate_insert_typed( &[String::from("payload")], &[Some(String::from("varbinary(max)"))], &[vec![json!("O'Brien")]], "files", "dbo", &DatabaseType::SqlServer, None, ); assert_eq!(sql, "INSERT INTO [dbo].[files] ([payload]) VALUES\n(CONVERT(varbinary(max), N'O''Brien'))"); } #[test] fn dameng_insert_formats_bit_booleans_as_numeric_literals() { let sql = generate_insert_typed( &[String::from("enabled"), String::from("deleted"), String::from("optional")], &[Some(String::from("BIT")), Some(String::from("bit")), Some(String::from("BIT"))], &[vec![json!(true), json!(false), serde_json::Value::Null]], "flags", "DBX_TEST", &DatabaseType::Dameng, None, ); assert_eq!( sql, "INSERT INTO \"DBX_TEST\".\"flags\" (\"enabled\", \"deleted\", \"optional\") VALUES\n(1, 0, NULL)" ); assert!(!sql.contains("TRUE")); assert!(!sql.contains("FALSE")); } #[test] fn sqlserver_upsert_formats_bit_booleans_as_numeric_literals() { let sql = generate_upsert_typed( &[String::from("id"), String::from("enabled")], &[Some(String::from("int")), Some(String::from("bit"))], &[vec![json!(1), json!(true)]], "flags", "dbo", &DatabaseType::SqlServer, &[String::from("id")], None, ); assert!(sql.contains("USING (VALUES\n(1, 1)\n)")); assert!(!sql.contains("TRUE")); assert!(!sql.contains("FALSE")); } #[test] fn postgres_insert_preserves_json_escape_sequences() { let sql = generate_insert_typed( &[String::from("payload")], &[Some(String::from("jsonb"))], &[vec![json!(r#"{"message":"hello\nworld"}"#)]], "events", "public", &DatabaseType::Postgres, None, ); assert_eq!( sql, r#"INSERT INTO "public"."events" ("payload") VALUES (E'{"message":"hello\\nworld"}')"# ); } #[test] fn postgres_insert_preserves_text_backslashes() { let sql = generate_insert_typed( &[String::from("path")], &[Some(String::from("text"))], &[vec![json!(r#"C:\tmp\file.txt"#)]], "files", "public", &DatabaseType::Postgres, None, ); assert_eq!( sql, r#"INSERT INTO "public"."files" ("path") VALUES (E'C:\\tmp\\file.txt')"# ); } #[test] fn postgres_insert_preserves_pgvector_literals_and_array_controls() { let sql = generate_insert_typed( &[ String::from("embedding"), String::from("qualified_embedding"), String::from("compact_embedding"), String::from("scores"), String::from("embedding_history"), ], &[ Some(String::from("vector(3)")), Some(String::from("public.vector(3)")), Some(String::from("extensions.halfvec(3)")), Some(String::from("real[]")), Some(String::from("vector(3)[]")), ], &[vec![ json!([1.25, -2.5, 3.75]), json!([0, 0.875, -0.014]), json!(["0.5", "-0.25", "4"]), json!([1.25, -2.5, 3.75]), json!([[1.25, -2.5, 3.75], [0, 0.875, -0.014]]), ]], "pgvector_probe", "target_7955", &DatabaseType::Postgres, None, ); assert_eq!( sql, r#"INSERT INTO "target_7955"."pgvector_probe" ("embedding", "qualified_embedding", "compact_embedding", "scores", "embedding_history") VALUES ('[1.25,-2.5,3.75]', '[0,0.875,-0.014]', '[0.5,-0.25,4]', '{1.25,-2.5,3.75}', '{{1.25,-2.5,3.75},{0,0.875,-0.014}}')"# ); } #[test] fn postgres_vector_literal_keeps_null_and_preformatted_string_behavior() { assert_eq!(escape_value_typed(&serde_json::Value::Null, &DatabaseType::Postgres, Some("vector(3)")), "NULL"); assert_eq!(escape_value_typed(&json!("[1,2,3]"), &DatabaseType::Postgres, Some("halfvec(3)")), "'[1,2,3]'"); assert_eq!(escape_value_typed(&json!([1, 2, 3]), &DatabaseType::Postgres, Some("integer[]")), "'{1,2,3}'"); } #[test] fn postgres_insert_formats_bytea_prefixed_hex_as_binary_literal() { let sql = generate_insert_typed( &[String::from("id"), String::from("payload"), String::from("note")], &[Some(String::from("integer")), Some(String::from("BYTEA")), Some(String::from("text"))], &[vec![json!(1), json!("0x48656c6c6f"), json!("0x48656c6c6f")]], "files", "public", &DatabaseType::Postgres, None, ); assert_eq!( sql, r#"INSERT INTO "public"."files" ("id", "payload", "note") VALUES (1, decode('48656c6c6f', 'hex'), '0x48656c6c6f')"# ); } #[test] fn mysql_insert_formats_blob_prefixed_hex_as_binary_literal() { let sql = generate_insert_typed( &[String::from("id"), String::from("payload"), String::from("empty_blob"), String::from("note")], &[ Some(String::from("int")), Some(String::from("MEDIUMBLOB")), Some(String::from("blob")), Some(String::from("varchar(64)")), ], &[vec![json!(1), json!("0x0001ABff"), json!("0X"), json!("0x0001ABff")]], "files", "", &DatabaseType::Mysql, None, ); assert_eq!( sql, r#"INSERT INTO `files` (`id`, `payload`, `empty_blob`, `note`) VALUES (1, 0x0001ABff, X'', '0x0001ABff')"# ); } #[test] fn mysql_insert_keeps_invalid_blob_hex_as_string_literal() { let sql = generate_insert_typed( &[String::from("id"), String::from("payload")], &[Some(String::from("int")), Some(String::from("mediumblob"))], &[vec![json!(1), json!("0xnothex")]], "files", "", &DatabaseType::Mysql, None, ); assert_eq!( sql, r#"INSERT INTO `files` (`id`, `payload`) VALUES (1, '0xnothex')"# ); } #[test] fn mysql_insert_keeps_backslash_escape_style() { let sql = generate_insert_typed( &[String::from("path")], &[Some(String::from("varchar(255)"))], &[vec![json!(r#"C:\tmp\file.txt"#)]], "files", "", &DatabaseType::Mysql, None, ); assert_eq!( sql, r#"INSERT INTO `files` (`path`) VALUES ('C:\\tmp\\file.txt')"# ); } #[test] fn postgres_insert_escapes_control_characters_quotes_and_backslashes() { let sql = generate_insert_typed( &[ String::from("line_break"), String::from("carriage_return"), String::from("quote"), String::from("path"), String::from("plain"), ], &[ Some(String::from("text")), Some(String::from("text")), Some(String::from("text")), Some(String::from("text")), Some(String::from("text")), ], &[vec![json!("line1\nline2"), json!("line1\rline2"), json!("O'Hara"), json!(r"C:\tmp"), json!("plain")]], "notes", "public", &DatabaseType::Postgres, None, ); assert_eq!( sql, r#"INSERT INTO "public"."notes" ("line_break", "carriage_return", "quote", "path", "plain") VALUES (E'line1\nline2', E'line1\rline2', 'O''Hara', E'C:\\tmp', 'plain')"# ); } #[test] fn postgres_insert_formats_whole_number_values_as_integer_literals() { let sql = generate_insert_typed( &[String::from("small_value"), String::from("integer_value"), String::from("big_value")], &[Some(String::from("smallint")), Some(String::from("integer")), Some(String::from("bigint"))], &[vec![json!(1.0), json!(-2.0), json!(3.0)]], "numbers", "public", &DatabaseType::Postgres, None, ); assert_eq!( sql, "INSERT INTO \"public\".\"numbers\" (\"small_value\", \"integer_value\", \"big_value\") VALUES\n(1, -2, 3)" ); } #[test] fn postgres_integer_literal_normalization_preserves_non_whole_and_out_of_range_values() { let scientific: serde_json::Value = serde_json::from_str("1e20").unwrap(); let out_of_range: serde_json::Value = serde_json::from_str("9223372036854775808.0").unwrap(); assert_eq!(escape_value_typed(&json!("1.0"), &DatabaseType::Postgres, Some("smallint")), "1"); assert_eq!(escape_value_typed(&json!("-2.0"), &DatabaseType::Postgres, Some("integer")), "-2"); assert_eq!(escape_value_typed(&json!("3.0"), &DatabaseType::Postgres, Some("bigint")), "3"); assert_eq!(escape_value_typed(&json!(1.25), &DatabaseType::Postgres, Some("smallint")), "1.25"); assert_eq!(escape_value_typed(&json!("1.25"), &DatabaseType::Postgres, Some("smallint")), "'1.25'"); assert_eq!(escape_value_typed(&scientific, &DatabaseType::Postgres, Some("bigint")), "1e+20"); assert_eq!(escape_value_typed(&json!("1e3"), &DatabaseType::Postgres, Some("bigint")), "'1e3'"); assert_eq!(escape_value_typed(&out_of_range, &DatabaseType::Postgres, Some("bigint")), "9223372036854775808.0"); assert_eq!( escape_value_typed(&json!("9223372036854775808.0"), &DatabaseType::Postgres, Some("bigint")), "'9223372036854775808.0'" ); assert_eq!(escape_value_typed(&json!("32768.0"), &DatabaseType::Postgres, Some("smallint")), "'32768.0'"); assert_eq!( escape_value_typed(&json!("2147483648.0"), &DatabaseType::Postgres, Some("integer")), "'2147483648.0'" ); assert_eq!(escape_value_typed(&json!("1.0"), &DatabaseType::Postgres, Some("text")), "'1.0'"); assert_eq!(escape_value_typed(&json!("1.0"), &DatabaseType::Postgres, Some("numeric")), "'1.0'"); assert_eq!(escape_value_typed(&json!("1.0"), &DatabaseType::Postgres, None), "'1.0'"); } #[test] fn oracle_single_row_insert_keeps_values_shape() { let sql = generate_insert_typed( &[String::from("id"), String::from("name")], &[Some(String::from("number")), Some(String::from("varchar2(64)"))], &[vec![json!(1), json!("Ada")]], "INSTR_CATEGORY", "APP", &DatabaseType::Oracle, None, ); assert_eq!( sql, r#"INSERT INTO "APP"."INSTR_CATEGORY" ("id", "name") VALUES (1, 'Ada')"# ); } #[test] fn oracle_multi_row_insert_uses_insert_all() { let sql = generate_insert_typed( &[String::from("id"), String::from("name")], &[Some(String::from("number")), Some(String::from("varchar2(64)"))], &[vec![json!(1), json!("Ada")], vec![json!(2), json!("O'Brien")]], "INSTR_CATEGORY", "APP", &DatabaseType::Oracle, None, ); assert_eq!( sql, r#"INSERT ALL INTO "APP"."INSTR_CATEGORY" ("id", "name") VALUES (1, 'Ada') INTO "APP"."INSTR_CATEGORY" ("id", "name") VALUES (2, 'O''Brien') SELECT 1 FROM dual"# ); } #[test] fn oracle_transfer_write_batches_limit_insert_all_rows() { let rows = (0..(MAX_ORACLE_INSERT_ALL_ROWS + 1)).map(|index| vec![json!(index)]).collect::>(); let statements = generate_transfer_write_sql_batches( &TransferMode::Append, &[String::from("id")], &[Some(String::from("number"))], &rows, "INSTR_CATEGORY", "APP", &DatabaseType::Oracle, &[], None, false, false, ) .unwrap(); assert_eq!(statements.len(), 2); assert_eq!(statements[0].matches("\nINTO ").count(), MAX_ORACLE_INSERT_ALL_ROWS); assert!(statements[0].starts_with("INSERT ALL\nINTO ")); assert!(statements[0].ends_with("SELECT 1 FROM dual")); } #[test] fn transfer_write_sql_batches_split_large_insert_statements() { let rows = (0..4).map(|index| vec![json!(index), json!("x".repeat(180 * 1024))]).collect::>(); let statements = generate_transfer_write_sql_batches( &TransferMode::Append, &[String::from("id"), String::from("payload")], &[Some(String::from("int")), Some(String::from("text"))], &rows, "events", "", &DatabaseType::Mysql, &[], None, false, false, ) .unwrap(); assert!(statements.len() > 1); assert!(statements.iter().all(|sql| sql.starts_with("INSERT INTO `events`"))); } #[test] fn mysql_sql_batch_allows_one_row_over_soft_target() { let rows = vec![vec![json!("x".repeat(256))]]; let limits = SqlBatchLimits { max_rows: 100, target_sql_bytes: 128, hard_sql_bytes: Some(1024) }; let batches = generate_insert_typed_sql_batches( &[String::from("payload")], &[Some(String::from("text"))], &rows, "events", "", &DatabaseType::Mysql, None, limits, ) .unwrap(); assert_eq!(batches.len(), 1); assert_eq!(batches[0].1, 1); } #[test] fn mysql_sql_batch_rejects_one_row_over_known_hard_limit() { let rows = vec![vec![json!("x".repeat(256))]]; let limits = SqlBatchLimits { max_rows: 100, target_sql_bytes: 128, hard_sql_bytes: Some(200) }; let error = generate_insert_typed_sql_batches( &[String::from("payload")], &[Some(String::from("text"))], &rows, "events", "", &DatabaseType::Mysql, None, limits, ) .unwrap_err(); assert!(error.contains("row 1")); assert!(error.contains("200 byte hard limit")); } #[test] fn sqlserver_insert_batches_enforce_values_row_limit() { let rows = (0..(MAX_SQLSERVER_INSERT_ROWS + 1)).map(|index| vec![json!(index)]).collect::>(); let batches = generate_insert_typed_sql_batches( &[String::from("id")], &[Some(String::from("int"))], &rows, "events", "dbo", &DatabaseType::SqlServer, None, SqlBatchLimits::for_database(&DatabaseType::SqlServer, rows.len()), ) .unwrap(); assert_eq!(batches.iter().map(|(_, row_count)| *row_count).collect::>(), vec![1000, 1]); } #[test] fn sqlserver_insert_batches_measure_unicode_sql_as_utf16() { let rows = (0..2).map(|_| vec![json!("x".repeat(140 * 1024))]).collect::>(); let batches = generate_insert_typed_sql_batches( &[String::from("payload")], &[Some(String::from("nvarchar(max)"))], &rows, "events", "dbo", &DatabaseType::SqlServer, None, SqlBatchLimits::for_database(&DatabaseType::SqlServer, rows.len()), ) .unwrap(); assert_eq!(batches.iter().map(|(_, row_count)| *row_count).collect::>(), vec![1, 1]); } #[test] fn sqlserver_sql_byte_count_uses_utf16_code_units() { assert_eq!(sql_text_bytes("AA\u{8d8a}\u{1f600}", &DatabaseType::SqlServer), 10); assert_eq!(sql_text_bytes("AA\u{8d8a}\u{1f600}", &DatabaseType::Postgres), 9); } #[test] fn transfer_write_sql_batches_keep_existing_upsert_sql_shape() { let statements = generate_transfer_write_sql_batches( &TransferMode::Upsert, &[String::from("id"), String::from("name")], &[Some(String::from("int")), Some(String::from("varchar(64)"))], &[vec![json!(1), json!("Ada")]], "users", "", &DatabaseType::Mysql, &[String::from("id")], None, false, false, ) .unwrap(); assert_eq!(statements.len(), 1); assert!(statements[0].contains("ON DUPLICATE KEY UPDATE")); } #[test] fn mysql_spatial_transfer_reuses_validated_wkb_markers_for_all_modes() { let columns = [String::from("id"), String::from("location"), String::from("name")]; let column_types = [Some(String::from("int")), Some(String::from("point")), Some(String::from("varchar(32)"))]; let rows = [vec![json!(1), json!("DBX_WKB:4326:0101000000000000000000F03F0000000000000040"), json!("alpha")]]; for mode in [TransferMode::Append, TransferMode::Overwrite, TransferMode::Upsert] { let statements = generate_transfer_write_sql_batches( &mode, &columns, &column_types, &rows, "places", "", &DatabaseType::Mysql, &[String::from("id")], None, false, true, ) .unwrap(); assert_eq!(statements.len(), 1); assert!(statements[0].contains("ST_GeomFromWKB(0x0101000000000000000000F03F0000000000000040, 4326)")); assert!(statements[0].contains("'alpha'")); if mode == TransferMode::Upsert { assert!(statements[0].contains("ON DUPLICATE KEY UPDATE")); } } } #[test] fn mysql_spatial_transfer_rejects_invalid_markers_and_keeps_public_insert_shape() { let invalid = json!("DBX_WKB:4326:0101000000"); let transfer = generate_transfer_write_sql_batches( &TransferMode::Append, &[String::from("location")], &[Some(String::from("point"))], &[vec![invalid.clone()]], "places", "", &DatabaseType::Mysql, &[], None, false, true, ) .unwrap(); let public = generate_insert_typed( &[String::from("location")], &[Some(String::from("point"))], &[vec![invalid]], "places", "", &DatabaseType::Mysql, None, ); assert_eq!(transfer, vec!["INSERT INTO `places` (`location`) VALUES\n('DBX_WKB:4326:0101000000')"]); assert_eq!(public, transfer[0]); assert!(!transfer[0].contains("ST_GeomFromWKB")); } #[test] fn mysql_spatial_transfer_projection_is_native_mysql_to_mysql_only() { let sql = "SELECT `id`, `location` FROM `places` ORDER BY `id` LIMIT 2 OFFSET 0".to_string(); let columns = [String::from("id"), String::from("location")]; let column_types = [Some(String::from("int")), Some(String::from("point"))]; let (native_sql, native_markers) = mysql_spatial_transfer_select_sql( sql.clone(), &columns, &column_types, &DatabaseType::Mysql, &DatabaseType::Mysql, ); assert!(native_markers); assert!(native_sql.contains("CONCAT('DBX_WKB:', ST_SRID(`location`), ':', HEX(ST_AsWKB(`location`)))")); assert!(native_sql.ends_with("ORDER BY `id` LIMIT 2 OFFSET 0")); let nonspatial_types = [Some(String::from("int")), Some(String::from("varchar(32)"))]; assert_eq!( mysql_spatial_transfer_select_sql( sql.clone(), &columns, &nonspatial_types, &DatabaseType::Mysql, &DatabaseType::Mysql, ), (sql.clone(), false) ); for (source, target) in [ (DatabaseType::Mysql, DatabaseType::Postgres), (DatabaseType::Postgres, DatabaseType::Mysql), (DatabaseType::Doris, DatabaseType::Mysql), ] { assert_eq!( mysql_spatial_transfer_select_sql(sql.clone(), &columns, &column_types, &source, &target), (sql.clone(), false) ); } } #[test] fn postgres_transfer_insert_overrides_generated_always_identity_values() { for mode in [TransferMode::Append, TransferMode::Overwrite] { let statements = generate_transfer_write_sql_batches( &mode, &[String::from("id"), String::from("name")], &[Some(String::from("bigint")), Some(String::from("text"))], &[vec![json!(42), json!("Ada")]], "users", "public", &DatabaseType::Postgres, &[], None, true, false, ) .unwrap(); assert_eq!(statements.len(), 1); assert_eq!( statements[0], "INSERT INTO \"public\".\"users\" (\"id\", \"name\") OVERRIDING SYSTEM VALUE VALUES\n(42, 'Ada')" ); } } #[test] fn postgres_transfer_upsert_overrides_generated_always_identity_values() { let statements = generate_transfer_write_sql_batches( &TransferMode::Upsert, &[String::from("id"), String::from("name")], &[Some(String::from("bigint")), Some(String::from("text"))], &[vec![json!(42), json!("Ada")]], "users", "public", &DatabaseType::Postgres, &[String::from("id")], None, true, false, ) .unwrap(); assert_eq!(statements.len(), 1); assert_eq!( statements[0], "INSERT INTO \"public\".\"users\" (\"id\", \"name\") OVERRIDING SYSTEM VALUE VALUES\n(42, 'Ada')\nON CONFLICT (\"id\") DO UPDATE SET \"name\" = EXCLUDED.\"name\"" ); } #[test] fn postgres_transfer_without_generated_always_identity_keeps_sql_shape() { let statements = generate_transfer_write_sql_batches( &TransferMode::Append, &[String::from("name")], &[Some(String::from("text"))], &[vec![json!("Ada")]], "users", "public", &DatabaseType::Postgres, &[], None, false, false, ) .unwrap(); assert_eq!(statements, vec!["INSERT INTO \"public\".\"users\" (\"name\") VALUES\n('Ada')"]); } #[test] fn postgres_system_value_override_is_not_applied_to_other_dialects() { let statements = generate_transfer_write_sql_batches( &TransferMode::Append, &[String::from("id")], &[Some(String::from("bigint"))], &[vec![json!(42)]], "users", "public", &DatabaseType::Kingbase, &[], None, true, false, ) .unwrap(); assert_eq!(statements, vec!["INSERT INTO \"public\".\"users\" (\"id\") VALUES\n(42)"]); } #[test] fn postgres_non_transfer_insert_keeps_existing_sql_shape() { let sql = generate_insert( &[String::from("id"), String::from("name")], &[vec![json!(42), json!("Ada")]], "users", "public", &DatabaseType::Postgres, ); assert_eq!(sql, "INSERT INTO \"public\".\"users\" (\"id\", \"name\") VALUES\n(42, 'Ada')"); } #[test] fn database_from_pool_key_handles_session_scoped_keys() { assert_eq!(database_from_pool_key("conn:analytics"), Some("analytics")); assert_eq!(database_from_pool_key("conn:analytics:session:editor-1"), Some("analytics")); assert_eq!(database_from_pool_key("conn"), None); assert_eq!(database_from_pool_key("conn:analytics:catalog:hive"), Some("analytics")); assert_eq!(database_from_pool_key("conn:catalog:hive"), None); assert_eq!(catalog_from_pool_key("conn:catalog:hive"), Some("hive")); assert_eq!(catalog_from_pool_key("conn:ads:catalog:paimon"), Some("paimon")); assert_eq!(catalog_from_pool_key("conn:ads"), None); } #[test] fn resolve_external_transfer_catalog_skips_builtin_catalogs() { assert_eq!(resolve_external_transfer_catalog(Some("hive"), &DatabaseType::StarRocks), Some("hive")); assert_eq!(resolve_external_transfer_catalog(Some("default_catalog"), &DatabaseType::StarRocks), None); assert_eq!(resolve_external_transfer_catalog(Some("internal"), &DatabaseType::Doris), None); // MySQL db_type is included so StarRocks saved as mysql+driver_profile still // gets 3-part catalog qualification (UI only sends catalog for capable engines). assert_eq!(resolve_external_transfer_catalog(Some("hive"), &DatabaseType::Mysql), Some("hive")); assert_eq!(resolve_external_transfer_catalog(Some("hive"), &DatabaseType::Postgres), None); } #[test] fn resolve_external_transfer_catalog_for_config_accepts_starrocks_driver_profile() { let config = crate::models::connection::ConnectionConfig { docs_notes_path: None, id: "sr".to_string(), name: "sr".to_string(), note: String::new(), db_type: DatabaseType::Mysql, driver_profile: Some("starrocks".to_string()), driver_label: None, url_params: None, agent_java_options: Vec::new(), host: "127.0.0.1".to_string(), port: 9030, username: String::new(), password: String::new(), database: None, default_schema: None, visible_databases: None, visible_database_patterns: None, visible_schemas: None, show_system_schemas: false, attached_databases: Vec::new(), init_script: None, color: None, transport_layers: Vec::new(), connect_timeout_secs: 5, query_timeout_secs: 30, idle_timeout_secs: 60, keepalive_interval_secs: 0, ssl: false, ca_cert_path: String::new(), client_cert_path: String::new(), client_key_path: String::new(), sysdba: false, oracle_connection_type: None, connection_string: None, redis_connection_mode: None, redis_sentinel_master: String::new(), redis_sentinel_nodes: String::new(), redis_sentinel_username: String::new(), redis_sentinel_password: String::new(), redis_sentinel_tls: false, redis_cluster_nodes: String::new(), redis_key_separator: crate::models::connection::default_redis_key_separator(), redis_scan_page_size: None, redis_database_aliases: Default::default(), redis_key_templates: Vec::new(), etcd_endpoints: String::new(), gbase_server: String::new(), informix_server: String::new(), external_config: None, jdbc_driver_class: None, jdbc_driver_paths: Vec::new(), one_time: false, save_password: true, read_only: false, is_production: false, production_databases: vec![], database_info: None, }; let mut gbase_jdbc = config.clone(); gbase_jdbc.db_type = DatabaseType::Jdbc; gbase_jdbc.connection_string = Some("jdbc:gbase://127.0.0.1:5258/dbx_test".to_string()); gbase_jdbc.jdbc_driver_class = Some("cn.gbase.Driver".to_string()); assert_eq!(effective_transfer_database_type(&gbase_jdbc), DatabaseType::Gbase); assert_eq!( transfer_object_family(&effective_transfer_database_type(&gbase_jdbc)), Some(TransferObjectFamily::Mysql) ); gbase_jdbc.connection_string = Some("jdbc:unknown://127.0.0.1:1234/dbx_test".to_string()); gbase_jdbc.jdbc_driver_class = Some("com.example.Driver".to_string()); assert_eq!(effective_transfer_database_type(&gbase_jdbc), DatabaseType::Jdbc); assert_eq!(resolve_external_transfer_catalog_for_config(Some("paimon"), &config), Some("paimon")); assert_eq!(resolve_external_transfer_catalog_for_config(Some("default_catalog"), &config), None); } #[test] fn transfer_read_retry_policy_retries_connection_errors() { assert_eq!( transfer_pool_error_action( TransferExecutionSafety::ReadOnlyRetryable, Some(DatabaseType::Postgres), "connection reset by peer" ), PoolErrorAction::ReconnectAndRetry ); } #[test] fn transfer_write_retry_policy_discards_without_replaying_batch() { assert_eq!( transfer_pool_error_action( TransferExecutionSafety::WriteNoReplay, Some(DatabaseType::Postgres), "connection reset by peer" ), PoolErrorAction::Discard ); } #[test] fn map_column_type_oracle_char_semantics_to_mysql() { // Issue #6479: Oracle `VARCHAR2(50 CHAR)` must rewrite to `VARCHAR(50)`, // not `VARCHAR(50 char)` which MySQL rejects with ERROR 1064 (42000). assert_eq!(map_column_type("VARCHAR2(6 CHAR)", &DatabaseType::Oracle, &DatabaseType::Mysql), "VARCHAR(6)"); assert_eq!(map_column_type("VARCHAR2(50 CHAR)", &DatabaseType::Oracle, &DatabaseType::Mysql), "VARCHAR(50)"); assert_eq!(map_column_type("VARCHAR2(20 char)", &DatabaseType::Oracle, &DatabaseType::Mysql), "VARCHAR(20)"); assert_eq!(map_column_type("VARCHAR2(50 CHAR)", &DatabaseType::Oracle, &DatabaseType::Mysql), "VARCHAR(50)"); // NVARCHAR2 keeps its pre-existing fallback (TEXT) — no length unit leaks. assert_eq!(map_column_type("NVARCHAR2(50 CHAR)", &DatabaseType::Oracle, &DatabaseType::Mysql), "TEXT"); } #[test] fn map_column_type_oracle_plain_varchar2_to_mysql() { // Plain VARCHAR2 without a length unit must keep working unchanged. assert_eq!(map_column_type("VARCHAR2(50)", &DatabaseType::Oracle, &DatabaseType::Mysql), "VARCHAR(50)"); assert_eq!(map_column_type("VARCHAR2", &DatabaseType::Oracle, &DatabaseType::Mysql), "VARCHAR(255)"); } #[test] fn map_column_type_preserves_length_units_for_oracle_family_targets() { for target in [DatabaseType::Oracle, DatabaseType::OceanbaseOracle, DatabaseType::Dameng] { let source = if target == DatabaseType::Oracle { DatabaseType::OceanbaseOracle } else { DatabaseType::Oracle }; assert_eq!(map_column_type("VARCHAR2(50 CHAR)", &source, &target), "VARCHAR(50 char)"); assert_eq!(map_column_type("CHAR(20 BYTE)", &source, &target), "CHAR(20 byte)"); } } #[test] fn map_column_type_strips_length_units_for_non_oracle_targets() { for target in [DatabaseType::Mysql, DatabaseType::Postgres, DatabaseType::SqlServer] { assert!(!map_column_type("VARCHAR2(50 CHAR)", &DatabaseType::Oracle, &target).contains("char")); assert!(!map_column_type("CHAR(20 BYTE)", &DatabaseType::Oracle, &target).contains("byte")); } } #[test] fn map_column_type_keeps_real_char_type_untouched() { // CHAR(10) is a valid type on its own and must not lose its length. assert_eq!(map_column_type("CHAR(10)", &DatabaseType::Oracle, &DatabaseType::Mysql), "CHAR(10)"); assert_eq!(map_column_type("char(10)", &DatabaseType::Mysql, &DatabaseType::Postgres), "CHAR(10)"); } #[test] fn map_column_type_oracle_byte_semantics_locked() { // MySQL has no BYTE length semantics. The qualifier is stripped so the // generated DDL stays valid; `VARCHAR2(20 BYTE)` maps to `VARCHAR(20)` // (character length). This intentionally locks in the conservative // behavior: never emit `VARCHAR(20 byte)`. assert_eq!(map_column_type("VARCHAR2(20 BYTE)", &DatabaseType::Oracle, &DatabaseType::Mysql), "VARCHAR(20)"); assert_eq!(map_column_type("VARCHAR2(20 byte)", &DatabaseType::Oracle, &DatabaseType::Mysql), "VARCHAR(20)"); } #[test] fn map_column_type_preserves_numeric_precision_scale() { // Precision/scale parameter lists must not be disturbed. assert_eq!(map_column_type("NUMBER(10,2)", &DatabaseType::Oracle, &DatabaseType::Mysql), "DECIMAL(10,2)"); assert_eq!(map_column_type("DECIMAL(10,2)", &DatabaseType::Oracle, &DatabaseType::Mysql), "DECIMAL(10,2)"); assert_eq!(map_column_type("NUMBER(10)", &DatabaseType::Oracle, &DatabaseType::Mysql), "DECIMAL(10)"); assert_eq!(map_column_type("TIMESTAMP(6)", &DatabaseType::Oracle, &DatabaseType::Mysql), "DATETIME"); } #[test] fn map_column_type_preserves_longtext_for_mysql_target() { assert_eq!(map_column_type("longtext", &DatabaseType::Mysql, &DatabaseType::Mysql), "longtext"); } #[test] fn map_column_type_preserves_mediumtext_for_mysql_target() { assert_eq!(map_column_type("mediumtext", &DatabaseType::Mysql, &DatabaseType::Mysql), "mediumtext"); } #[test] fn map_column_type_preserves_longblob_for_mysql_target() { assert_eq!(map_column_type("longblob", &DatabaseType::Mysql, &DatabaseType::Mysql), "longblob"); } #[test] fn map_column_type_preserves_mediumblob_for_mysql_target() { assert_eq!(map_column_type("mediumblob", &DatabaseType::Mysql, &DatabaseType::Mysql), "mediumblob"); } #[test] fn map_column_type_preserves_same_database_type() { assert_eq!(map_column_type("int unsigned", &DatabaseType::Mysql, &DatabaseType::Mysql), "int unsigned"); assert_eq!( map_column_type("int unsigned zerofill", &DatabaseType::Mysql, &DatabaseType::Mysql), "int unsigned zerofill" ); assert_eq!(map_column_type("bigint unsigned", &DatabaseType::Mysql, &DatabaseType::Mysql), "bigint unsigned"); assert_eq!( map_column_type("bigint unsigned zerofill", &DatabaseType::Mysql, &DatabaseType::Mysql), "bigint unsigned zerofill" ); } #[test] fn map_column_type_preserves_numeric_type_from_mysql_to_postgres() { assert_eq!(map_column_type("int unsigned", &DatabaseType::Mysql, &DatabaseType::Postgres), "INTEGER"); assert_eq!(map_column_type("int unsigned zerofill", &DatabaseType::Mysql, &DatabaseType::Postgres), "INTEGER"); assert_eq!(map_column_type("bigint unsigned", &DatabaseType::Mysql, &DatabaseType::Postgres), "BIGINT"); assert_eq!( map_column_type("bigint unsigned zerofill", &DatabaseType::Mysql, &DatabaseType::Postgres), "BIGINT" ); } #[test] fn map_column_type_longtext_falls_back_to_text_for_non_mysql_target() { assert_eq!(map_column_type("longtext", &DatabaseType::Mysql, &DatabaseType::Postgres), "TEXT"); } #[test] fn map_column_type_longblob_falls_back_for_non_mysql_target() { assert_eq!(map_column_type("longblob", &DatabaseType::Mysql, &DatabaseType::Postgres), "BYTEA"); } #[test] fn parse_mysql_row_error_extracts_row_number() { let err = "ERROR 22001 (1406): Data too long column 'content' at row 8"; assert_eq!(parse_mysql_row_error(err), Some(8)); } #[test] fn parse_mysql_row_error_returns_none_for_non_mysql_error() { assert_eq!(parse_mysql_row_error("some other error"), None); } #[test] fn mysql_create_table_preserves_auto_increment_primary_key() { let cols = vec![ db::ColumnInfo { is_primary_key: true, is_nullable: false, extra: Some("auto_increment".to_string()), ..test_column("id", "INT") }, db::ColumnInfo { is_nullable: false, ..test_column("name", "varchar(64)") }, ]; let ddl = generate_create_table_ddl(&cols, "users", "", "", &DatabaseType::Mysql, &DatabaseType::Mysql, None, None); assert!(ddl.contains("`id` INT NOT NULL AUTO_INCREMENT"), "ddl: {ddl}"); assert!(ddl.contains("PRIMARY KEY (`id`)"), "ddl: {ddl}"); } #[test] fn mysql_create_table_preserves_numeric_default_zero() { let cols = vec![db::ColumnInfo { is_nullable: false, column_default: Some("0".to_string()), ..test_column("status", "tinyint") }]; let ddl = generate_create_table_ddl(&cols, "items", "", "", &DatabaseType::Mysql, &DatabaseType::Mysql, None, None); assert!(ddl.contains("DEFAULT 0"), "ddl: {ddl}"); assert!(!ddl.contains("'0'"), "ddl should not quote numeric default: {ddl}"); } #[test] fn mysql_create_table_quotes_string_default_with_escape() { let cols = vec![db::ColumnInfo { column_default: Some("o'clock".to_string()), ..test_column("label", "varchar(32)") }]; let ddl = generate_create_table_ddl(&cols, "items", "", "", &DatabaseType::Mysql, &DatabaseType::Mysql, None, None); assert!(ddl.contains("DEFAULT 'o''clock'"), "ddl: {ddl}"); } #[test] fn mysql_create_table_keeps_current_timestamp_default_and_on_update() { let cols = vec![db::ColumnInfo { is_nullable: false, column_default: Some("CURRENT_TIMESTAMP".to_string()), extra: Some("DEFAULT_GENERATED on update CURRENT_TIMESTAMP".to_string()), ..test_column("updated_at", "timestamp") }]; let ddl = generate_create_table_ddl(&cols, "items", "", "", &DatabaseType::Mysql, &DatabaseType::Mysql, None, None); assert!(ddl.contains("DEFAULT CURRENT_TIMESTAMP"), "ddl: {ddl}"); assert!(ddl.contains("ON UPDATE CURRENT_TIMESTAMP"), "ddl: {ddl}"); assert!(ddl.contains("NOT NULL"), "ddl: {ddl}"); assert!(!ddl.contains("DEFAULT_GENERATED"), "ddl should not leak DEFAULT_GENERATED: {ddl}"); } #[test] fn mysql_create_table_keeps_current_timestamp_with_fsp() { let cols = vec![db::ColumnInfo { is_nullable: false, column_default: Some("CURRENT_TIMESTAMP(6)".to_string()), ..test_column("created_at", "timestamp(6)") }]; let ddl = generate_create_table_ddl(&cols, "items", "", "", &DatabaseType::Mysql, &DatabaseType::Mysql, None, None); assert!(ddl.contains("DEFAULT CURRENT_TIMESTAMP(6)"), "ddl: {ddl}"); } #[test] fn mysql_create_table_emits_on_update_without_default() { let cols = vec![db::ColumnInfo { is_nullable: false, extra: Some("on update CURRENT_TIMESTAMP(3)".to_string()), ..test_column("touched_at", "timestamp(3)") }]; let ddl = generate_create_table_ddl(&cols, "items", "", "", &DatabaseType::Mysql, &DatabaseType::Mysql, None, None); assert!(ddl.contains("ON UPDATE CURRENT_TIMESTAMP(3)"), "ddl: {ddl}"); assert!(!ddl.contains("DEFAULT"), "ddl should not emit DEFAULT when none was set: {ddl}"); } #[test] fn non_mysql_target_does_not_emit_auto_increment() { let cols = vec![db::ColumnInfo { is_primary_key: true, is_nullable: false, extra: Some("auto_increment".to_string()), ..test_column("id", "int") }]; let ddl = generate_create_table_ddl(&cols, "users", "", "", &DatabaseType::Sqlite, &DatabaseType::Mysql, None, None); assert!(!ddl.contains("AUTO_INCREMENT"), "non-mysql target should not emit AUTO_INCREMENT: {ddl}"); } #[test] fn postgres_create_table_default_clause_unchanged() { let cols = vec![db::ColumnInfo { data_type: "integer".to_string(), column_default: Some("nextval('public.t_id_seq'::regclass)".to_string()), is_primary_key: true, is_nullable: false, ..test_column("id", "integer") }]; let ddl = generate_create_table_ddl( &cols, "t", "public", "public", &DatabaseType::Postgres, &DatabaseType::Postgres, None, None, ); assert!(ddl.contains("GENERATED BY DEFAULT AS IDENTITY"), "ddl: {ddl}"); } #[test] fn postgres_create_table_preserves_identity_from_column_extra() { let cols = vec![db::ColumnInfo { data_type: "integer".to_string(), extra: Some("generated by default as identity".to_string()), is_primary_key: true, is_nullable: false, ..test_column("id", "integer") }]; let ddl = generate_create_table_ddl( &cols, "t", "public", "public", &DatabaseType::Postgres, &DatabaseType::Postgres, None, None, ); assert!(ddl.contains("\"id\" integer generated by default as identity NOT NULL"), "ddl: {ddl}"); } #[test] fn kingbase_transfer_uses_postgres_compatible_types() { assert_eq!(map_column_type("jsonb", &DatabaseType::Postgres, &DatabaseType::Kingbase), "JSONB"); assert_eq!(map_column_type("bytea", &DatabaseType::Postgres, &DatabaseType::Kingbase), "BYTEA"); assert_eq!(map_column_type("uuid", &DatabaseType::Postgres, &DatabaseType::Kingbase), "UUID"); assert_eq!(map_column_type("serial", &DatabaseType::Postgres, &DatabaseType::Kingbase), "SERIAL"); } #[test] fn kingbase_create_table_preserves_postgres_defaults() { let cols = vec![db::ColumnInfo { data_type: "integer".to_string(), column_default: Some("nextval('source.items_id_seq'::regclass)".to_string()), is_primary_key: true, is_nullable: false, ..test_column("id", "integer") }]; let ddl = generate_create_table_ddl( &cols, "items", "source", "target", &DatabaseType::Kingbase, &DatabaseType::Postgres, None, None, ); assert!(ddl.contains("GENERATED BY DEFAULT AS IDENTITY"), "ddl: {ddl}"); } #[test] fn kingbase_upsert_uses_on_conflict() { let sql = generate_upsert_typed( &[String::from("id"), String::from("name")], &[Some(String::from("integer")), Some(String::from("text"))], &[vec![json!(1), json!("updated")]], "items", "public", &DatabaseType::Kingbase, &[String::from("id")], None, ); assert!(sql.contains("ON CONFLICT (\"id\") DO UPDATE SET \"name\" = EXCLUDED.\"name\""), "sql: {sql}"); } #[test] fn kingbase_reused_ddl_uses_postgres_statement_sanitization() { let ddl = r#"CREATE TABLE public.items (id integer PRIMARY KEY); CREATE INDEX items_name_idx ON public.items (id);"#; let statements = transfer_ddl_statements(ddl, &DatabaseType::Kingbase); assert_eq!(statements.len(), 1); assert!(statements[0].starts_with("CREATE TABLE")); } #[test] fn transfer_progress_queue_has_bounded_capacity() { let (sender, receiver) = tokio::sync::mpsc::channel(TRANSFER_PROGRESS_CHANNEL_CAPACITY); for rows_transferred in 0..=TRANSFER_PROGRESS_CHANNEL_CAPACITY { try_send_transfer_progress( &sender, TransferProgress { transfer_id: "transfer".to_string(), table: "table".to_string(), table_index: 0, total_tables: 1, rows_transferred: rows_transferred as u64, total_rows: None, status: TransferStatus::Running, error: None, terminal: false, }, ); } assert_eq!(receiver.len(), TRANSFER_PROGRESS_CHANNEL_CAPACITY); } }