feat(transwarp): add Inceptor data source with table operations
This commit is contained in:
1 parent
597b134d21
commit
6fdc338348
101 files changed
+2300
-66
No files matched your search
@@ -1906,6 +1906,50 @@
|
||||
"driverManagement": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"dbType": "transwarp",
|
||||
"label": "星环Inceptor",
|
||||
"dialect": "Transwarp",
|
||||
"runtimeMode": "agent",
|
||||
"mcpMode": "bridge",
|
||||
"agentKey": "transwarp",
|
||||
"driverStoreVisible": true,
|
||||
"driverStoreOrder": 52,
|
||||
"driverProfiles": [
|
||||
{
|
||||
"profile": "transwarp-inceptor",
|
||||
"agentKey": "transwarp",
|
||||
"label": "星环Inceptor",
|
||||
"storeVisible": false
|
||||
}
|
||||
],
|
||||
"singleConnectionPool": false,
|
||||
"metadataConnectionScoped": false,
|
||||
"skipTcpProbe": true,
|
||||
"defaultPort": 10000,
|
||||
"traits": {
|
||||
"schemaAware": true
|
||||
},
|
||||
"supportLevel": "operate",
|
||||
"capabilities": {
|
||||
"queryExecution": true,
|
||||
"metadataBrowse": true,
|
||||
"objectBrowser": true,
|
||||
"objectSource": true,
|
||||
"schemaSearch": true,
|
||||
"diagram": false,
|
||||
"tableDataEdit": true,
|
||||
"tableStructureEdit": true,
|
||||
"tableImport": true,
|
||||
"dataTransfer": true,
|
||||
"sqlFileExecution": true,
|
||||
"databaseCreate": true,
|
||||
"fieldLineage": false,
|
||||
"sqlExplain": false,
|
||||
"userAdmin": false,
|
||||
"driverManagement": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"dbType": "kyuubi",
|
||||
"label": "Apache Kyuubi",
|
||||
|
||||
@@ -325,6 +325,7 @@ macro_rules! agent_connection_pool_database_type {
|
||||
| DatabaseType::Kyuubi
|
||||
| DatabaseType::Impala
|
||||
| DatabaseType::Argo
|
||||
| DatabaseType::Transwarp
|
||||
| DatabaseType::Spark
|
||||
| DatabaseType::Db2
|
||||
| DatabaseType::Informix
|
||||
|
||||
@@ -4891,7 +4891,7 @@ fn effective_import_batch_size(db_type: &DatabaseType, requested: usize) -> usiz
|
||||
let max_rows = match db_type {
|
||||
DatabaseType::Oracle | DatabaseType::OceanbaseOracle => MAX_ORACLE_IMPORT_BATCH_ROWS,
|
||||
DatabaseType::Iris => 1,
|
||||
DatabaseType::CloudflareD1 => 100,
|
||||
DatabaseType::CloudflareD1 | DatabaseType::Transwarp => 100,
|
||||
DatabaseType::SqlServer => 1000,
|
||||
DatabaseType::Sqlite => SQLITE_APPEND_COMMIT_ROWS,
|
||||
_ => usize::MAX,
|
||||
@@ -5204,6 +5204,7 @@ fn text_data_type(db_type: &DatabaseType) -> &'static str {
|
||||
DatabaseType::Oracle | DatabaseType::OceanbaseOracle | DatabaseType::Dameng => "CLOB",
|
||||
DatabaseType::ClickHouse => "String",
|
||||
DatabaseType::Hive
|
||||
| DatabaseType::Transwarp
|
||||
| DatabaseType::Kyuubi
|
||||
| DatabaseType::Trino
|
||||
| DatabaseType::PrestoSql
|
||||
@@ -12085,6 +12086,42 @@ mod tests {
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transwarp_import_batches_preserve_values_across_multiple_statements() {
|
||||
let mappings = vec!["id", "label", "payload"]
|
||||
.into_iter()
|
||||
.map(|name| TableImportColumnMapping {
|
||||
source_column: name.to_string(),
|
||||
target_column: name.to_string(),
|
||||
target_data_type: None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let data = ParsedImportFile {
|
||||
columns: vec!["id".into(), "label".into(), "payload".into()],
|
||||
rows: (0..205)
|
||||
.map(|index| {
|
||||
vec![
|
||||
serde_json::json!(index),
|
||||
if index == 101 { serde_json::Value::Null } else { serde_json::json!("dbx\u{4e2d}\u{6587}'s") },
|
||||
serde_json::json!(r#"{"key":[1,2]}"#),
|
||||
]
|
||||
})
|
||||
.collect(),
|
||||
total_rows: 205,
|
||||
effective_encoding: None,
|
||||
};
|
||||
let batches =
|
||||
build_import_insert_batches(&data, &mappings, &[], "events", "analytics", &DatabaseType::Transwarp, 500)
|
||||
.unwrap();
|
||||
assert_eq!(batches.iter().map(|batch| batch.row_count).collect::<Vec<_>>(), vec![100, 100, 5]);
|
||||
assert!(batches.iter().all(|batch| batch
|
||||
.sql
|
||||
.starts_with("INSERT INTO `analytics`.`events` (`id`, `label`, `payload`)\nSELECT ")));
|
||||
assert!(batches[0].sql.contains("dbx\u{4e2d}\u{6587}''s"));
|
||||
assert!(batches[1].sql.contains("SELECT 101, NULL"));
|
||||
assert!(batches[2].sql.contains("SELECT 204,"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn starrocks_csv_json_array_import_uses_typed_json_expression() {
|
||||
let plan = CompiledImportPlan {
|
||||
|
||||
@@ -121,6 +121,7 @@ impl SqlBatchLimits {
|
||||
let max_rows = requested_max_rows.max(1).min(match db_type {
|
||||
DatabaseType::SqlServer => MAX_SQLSERVER_INSERT_ROWS,
|
||||
DatabaseType::Oracle | DatabaseType::OceanbaseOracle => MAX_ORACLE_INSERT_ALL_ROWS,
|
||||
DatabaseType::Transwarp => 100,
|
||||
_ => usize::MAX,
|
||||
});
|
||||
let target_sql_bytes = match db_type {
|
||||
@@ -3439,7 +3440,10 @@ pub fn map_column_type(source_type: &str, source_db: &DatabaseType, target_db: &
|
||||
// Extract basic type, `bigint unsigned` -> `bigint`
|
||||
base = base.split(' ').next().unwrap_or(base).trim();
|
||||
|
||||
if matches!(target_db, DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala | DatabaseType::Argo) {
|
||||
if matches!(
|
||||
target_db,
|
||||
DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala | DatabaseType::Argo | DatabaseType::Transwarp
|
||||
) {
|
||||
return match base {
|
||||
"tinyint" => "TINYINT".into(),
|
||||
"smallint" | "int2" => "SMALLINT".into(),
|
||||
@@ -3684,7 +3688,11 @@ fn generate_create_table_ddl_with_column_quoting(
|
||||
if !c.is_nullable
|
||||
&& !matches!(
|
||||
target_db,
|
||||
DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala | DatabaseType::Argo
|
||||
DatabaseType::Hive
|
||||
| DatabaseType::Kyuubi
|
||||
| DatabaseType::Impala
|
||||
| DatabaseType::Argo
|
||||
| DatabaseType::Transwarp
|
||||
)
|
||||
{
|
||||
line.push_str(" NOT NULL");
|
||||
@@ -3709,7 +3717,10 @@ fn generate_create_table_ddl_with_column_quoting(
|
||||
}
|
||||
|
||||
let mut pks = Vec::with_capacity(columns.iter().filter(|c| c.is_primary_key).count());
|
||||
if !matches!(target_db, DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala | DatabaseType::Argo) {
|
||||
if !matches!(
|
||||
target_db,
|
||||
DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala | DatabaseType::Argo | DatabaseType::Transwarp
|
||||
) {
|
||||
for c in columns {
|
||||
if c.is_primary_key {
|
||||
let qname = transfer_column_identifier(&c.name, target_db, quote_target_column_names);
|
||||
@@ -3881,6 +3892,7 @@ pub(crate) fn generate_insert_typed_from_value_rows(
|
||||
struct InsertSqlTemplate {
|
||||
standard_prefix: String,
|
||||
oracle_into_prefix: Option<String>,
|
||||
inceptor_select_prefix: Option<String>,
|
||||
xugu_multirow_values: bool,
|
||||
}
|
||||
|
||||
@@ -3927,6 +3939,8 @@ impl InsertSqlTemplate {
|
||||
// round trip per row), which makes CSV imports orders of magnitude slower.
|
||||
oracle_into_prefix: matches!(db_type, DatabaseType::Oracle | DatabaseType::OceanbaseOracle)
|
||||
.then(|| format!("INTO {full_table} ({col_list}) VALUES ")),
|
||||
inceptor_select_prefix: (matches!(db_type, DatabaseType::Transwarp) && !columns.is_empty())
|
||||
.then(|| format!("INSERT INTO {full_table} ({col_list})\n")),
|
||||
// Xugu accepts consecutive row constructors (`VALUES (...) (...)`) but rejects
|
||||
// the comma-separated multi-row form emitted by the generic template.
|
||||
xugu_multirow_values: matches!(db_type, DatabaseType::Xugu),
|
||||
@@ -3937,6 +3951,22 @@ impl InsertSqlTemplate {
|
||||
if value_rows.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
if let Some(prefix) = self.inceptor_select_prefix.as_deref() {
|
||||
let mut sql = String::with_capacity(self.statement_bytes(
|
||||
value_rows.iter().map(String::len).sum(),
|
||||
value_rows.len(),
|
||||
&DatabaseType::Transwarp,
|
||||
));
|
||||
sql.push_str(prefix);
|
||||
for (index, values) in value_rows.iter().enumerate() {
|
||||
if index > 0 {
|
||||
sql.push_str("\nUNION ALL\n");
|
||||
}
|
||||
sql.push_str("SELECT ");
|
||||
sql.push_str(values.strip_prefix('(').and_then(|row| row.strip_suffix(')')).unwrap_or(values));
|
||||
}
|
||||
return sql;
|
||||
}
|
||||
if let Some(into_prefix) = self.oracle_into_prefix.as_deref().filter(|_| value_rows.len() > 1) {
|
||||
let capacity = "INSERT ALL\n".len()
|
||||
+ into_prefix.len().saturating_mul(value_rows.len())
|
||||
@@ -3975,6 +4005,12 @@ impl InsertSqlTemplate {
|
||||
}
|
||||
|
||||
fn statement_bytes(&self, value_rows_bytes: usize, row_count: usize, db_type: &DatabaseType) -> usize {
|
||||
if let Some(prefix) = self.inceptor_select_prefix.as_deref() {
|
||||
return sql_text_bytes(prefix, db_type)
|
||||
.saturating_add(value_rows_bytes.saturating_sub(2usize.saturating_mul(row_count)))
|
||||
.saturating_add("SELECT ".len().saturating_mul(row_count))
|
||||
.saturating_add("\nUNION ALL\n".len().saturating_mul(row_count.saturating_sub(1)));
|
||||
}
|
||||
if let Some(into_prefix) = self.oracle_into_prefix.as_deref().filter(|_| row_count > 1) {
|
||||
return sql_text_bytes("INSERT ALL\n", db_type)
|
||||
.saturating_add(sql_text_bytes(into_prefix, db_type).saturating_mul(row_count))
|
||||
@@ -4399,7 +4435,14 @@ fn generate_upsert_typed_for_transfer(
|
||||
fn max_transfer_write_rows(db_type: &DatabaseType, mode: &TransferMode) -> usize {
|
||||
match (db_type, mode) {
|
||||
(DatabaseType::SqlServer, TransferMode::Append | TransferMode::Overwrite) => MAX_SQLSERVER_INSERT_ROWS,
|
||||
(DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala | DatabaseType::Argo, _) => 500,
|
||||
(
|
||||
DatabaseType::Hive
|
||||
| DatabaseType::Kyuubi
|
||||
| DatabaseType::Impala
|
||||
| DatabaseType::Argo
|
||||
| DatabaseType::Transwarp,
|
||||
_,
|
||||
) => 500,
|
||||
(DatabaseType::Oracle, TransferMode::Append | TransferMode::Overwrite) => MAX_ORACLE_INSERT_ALL_ROWS,
|
||||
// The INSERT ALL template is shared with OceanBase's Oracle mode, so its
|
||||
// append/overwrite batches must clamp to the same per-statement row count.
|
||||
@@ -8798,6 +8841,45 @@ struct HiveServerTransferCursor {
|
||||
session_id: Option<String>,
|
||||
}
|
||||
|
||||
fn is_transwarp_complex_type(data_type: &str) -> bool {
|
||||
matches!(
|
||||
data_type.trim().split('<').next().unwrap_or("").trim().to_ascii_lowercase().as_str(),
|
||||
"array" | "map" | "struct" | "uniontype"
|
||||
)
|
||||
}
|
||||
|
||||
fn should_use_transwarp_server_side_copy(
|
||||
source_db_type: &DatabaseType,
|
||||
content: &TransferContent,
|
||||
columns: &[db::ColumnInfo],
|
||||
) -> bool {
|
||||
*source_db_type == DatabaseType::Transwarp
|
||||
&& should_copy_data(content)
|
||||
&& columns.iter().any(|column| is_transwarp_complex_type(&column.data_type))
|
||||
}
|
||||
|
||||
fn transwarp_server_side_copy_sql(
|
||||
source_columns: &[String],
|
||||
target_columns: &[String],
|
||||
source_table: &str,
|
||||
source_schema: &str,
|
||||
target_table: &str,
|
||||
target_schema: &str,
|
||||
quote_target_columns: bool,
|
||||
) -> String {
|
||||
let db_type = DatabaseType::Transwarp;
|
||||
let source = qualified_table(source_table, source_schema, &db_type, None);
|
||||
let target = qualified_table(target_table, target_schema, &db_type, None);
|
||||
let source_columns =
|
||||
source_columns.iter().map(|name| quote_identifier(name, &db_type)).collect::<Vec<_>>().join(", ");
|
||||
let target_columns = target_columns
|
||||
.iter()
|
||||
.map(|name| transfer_column_identifier(name, &db_type, quote_target_columns))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
format!("INSERT INTO {target} ({target_columns}) SELECT {source_columns} FROM {source}")
|
||||
}
|
||||
|
||||
fn transfer_cursor_sql(
|
||||
columns: &[String],
|
||||
table: &str,
|
||||
@@ -8826,7 +8908,7 @@ async fn fetch_hive_server_transfer_batch(
|
||||
};
|
||||
let pool_handle = state.pool_handle(pool_key).await;
|
||||
let Some(PoolKind::Agent(client)) = pool_handle.as_ref() else {
|
||||
return Err("Impala transfer requires an Agent connection".to_string());
|
||||
return Err("Agent cursor transfer requires an Agent connection".to_string());
|
||||
};
|
||||
let client = client.clone();
|
||||
|
||||
@@ -9616,6 +9698,26 @@ where
|
||||
};
|
||||
log::info!("[transfer] {} total_rows={:?}", table, total_rows);
|
||||
|
||||
let server_side_complex_copy =
|
||||
should_use_transwarp_server_side_copy(source_db_type, &request.content, &writable_columns);
|
||||
if server_side_complex_copy {
|
||||
if *target_db_type != DatabaseType::Transwarp || request.source_connection_id != request.target_connection_id {
|
||||
return Err(format!(
|
||||
"Native Inceptor complex columns in '{table}' require source and target on the same Transwarp connection"
|
||||
));
|
||||
}
|
||||
let source_table =
|
||||
qualified_table(table, &request.source_schema, source_db_type, request.source_catalog.as_deref());
|
||||
let destination_table =
|
||||
qualified_table(&target_table, &request.target_schema, target_db_type, request.target_catalog.as_deref());
|
||||
if source_table.eq_ignore_ascii_case(&destination_table) {
|
||||
return Err(format!("Cannot transfer native complex columns in '{table}' onto the same table"));
|
||||
}
|
||||
if total_rows.is_none() {
|
||||
return Err(format!("Cannot count native complex rows in '{table}' before transfer"));
|
||||
}
|
||||
}
|
||||
|
||||
// Create table on target if requested
|
||||
if request.create_table {
|
||||
create_transfer_target_table(
|
||||
@@ -9695,6 +9797,7 @@ where
|
||||
| DatabaseType::Kyuubi
|
||||
| DatabaseType::Impala
|
||||
| DatabaseType::Argo
|
||||
| DatabaseType::Transwarp
|
||||
))
|
||||
|| matches!(target_db_type, DatabaseType::Postgres | DatabaseType::Dameng | DatabaseType::H2);
|
||||
let target_columns = if needs_target_columns {
|
||||
@@ -9761,6 +9864,27 @@ where
|
||||
col_names.clone()
|
||||
};
|
||||
|
||||
if server_side_complex_copy {
|
||||
for (source_column, target_name) in writable_columns.iter().zip(&write_col_names) {
|
||||
let Some(target_column) =
|
||||
target_columns.iter().find(|column| column.name.eq_ignore_ascii_case(target_name))
|
||||
else {
|
||||
if target_table_preexisting {
|
||||
return Err(format!("Target table '{target_table}' is missing column '{target_name}'"));
|
||||
}
|
||||
continue;
|
||||
};
|
||||
if is_transwarp_complex_type(&source_column.data_type)
|
||||
&& !source_column.data_type.trim().eq_ignore_ascii_case(target_column.data_type.trim())
|
||||
{
|
||||
return Err(format!(
|
||||
"Native Inceptor column '{}' needs matching source and target types ({} vs {})",
|
||||
source_column.name, source_column.data_type, target_column.data_type
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Truncate target if overwrite mode (only when not rebuilding the table).
|
||||
// When drop_target_before_create is true, the target table was just created
|
||||
// and is already empty, so TRUNCATE is unnecessary.
|
||||
@@ -9785,6 +9909,7 @@ where
|
||||
| DatabaseType::Kyuubi
|
||||
| DatabaseType::Impala
|
||||
| DatabaseType::Argo
|
||||
| DatabaseType::Transwarp
|
||||
) {
|
||||
log::warn!("[transfer] upsert not supported for {:?}, falling back to append", target_db_type);
|
||||
(TransferMode::Append, vec![])
|
||||
@@ -9813,6 +9938,37 @@ where
|
||||
let mut offset: u64 = 0;
|
||||
let mut total_transferred: u64 = 0;
|
||||
|
||||
if server_side_complex_copy {
|
||||
if is_cancelled(&request.transfer_id).await {
|
||||
return Err("Cancelled".to_string());
|
||||
}
|
||||
let sql = transwarp_server_side_copy_sql(
|
||||
&col_names,
|
||||
&write_col_names,
|
||||
table,
|
||||
&request.source_schema,
|
||||
&target_table,
|
||||
&request.target_schema,
|
||||
request.quote_target_column_names,
|
||||
);
|
||||
execute_on_pool(state, target_pool_key, &sql)
|
||||
.await
|
||||
.map_err(|error| format!("Native Inceptor transfer failed for '{table}': {error}"))?;
|
||||
let copied = total_rows.expect("complex transfer count validated");
|
||||
progress_callback(TransferProgress {
|
||||
transfer_id: request.transfer_id.clone(),
|
||||
table: table.to_string(),
|
||||
table_index,
|
||||
total_tables,
|
||||
rows_transferred: copied,
|
||||
total_rows,
|
||||
status: TransferStatus::Running,
|
||||
error: None,
|
||||
terminal: false,
|
||||
});
|
||||
return Ok(copied);
|
||||
}
|
||||
|
||||
// COPY fast path: PG-family append/overwrite transfers stream the whole
|
||||
// table through the COPY protocol instead of paged SELECT + multi-row
|
||||
// INSERT — no per-batch statement parsing, no JSON round-trip. Any failure
|
||||
@@ -9895,9 +10051,11 @@ where
|
||||
// does not hold up mid-table.
|
||||
let mut keyset_indexes = transfer_keyset_column_indexes(&writable_columns, &primary_key_columns, source_db_type);
|
||||
let mut keyset_cursor: Vec<serde_json::Value> = Vec::new();
|
||||
// A single Agent cursor keeps Kyuubi/Impala rows in one query execution.
|
||||
// A single Agent cursor keeps Hive-family rows in one query execution. Inceptor
|
||||
// rejects the generic LIMIT/OFFSET form, just like the other Agent cursor paths.
|
||||
// Re-running LIMIT/OFFSET pages is unstable for tables without a unique key.
|
||||
let use_hive_server_cursor = matches!(source_db_type, DatabaseType::Kyuubi | DatabaseType::Impala);
|
||||
let use_hive_server_cursor =
|
||||
matches!(source_db_type, DatabaseType::Kyuubi | DatabaseType::Impala | DatabaseType::Transwarp);
|
||||
let hive_server_transfer_sql = use_hive_server_cursor.then(|| {
|
||||
transfer_cursor_sql(
|
||||
&col_names,
|
||||
@@ -17295,6 +17453,73 @@ SELECT 1 FROM dual"#
|
||||
assert_eq!(sql, "INSERT INTO \"public\".\"users\" (\"id\", \"name\") VALUES\n(42, 'Ada')");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inceptor_insert_uses_select_for_ordinary_tables() {
|
||||
assert_eq!(SqlBatchLimits::for_database(&DatabaseType::Transwarp, 500).max_rows, 100);
|
||||
let sql = generate_insert_typed(
|
||||
&[String::from("id"), String::from("label")],
|
||||
&[Some(String::from("int")), Some(String::from("string"))],
|
||||
&[vec![json!(1), json!("first")], vec![json!(2), json!("second")]],
|
||||
"events",
|
||||
"analytics",
|
||||
&DatabaseType::Transwarp,
|
||||
None,
|
||||
);
|
||||
assert_eq!(
|
||||
sql,
|
||||
"INSERT INTO `analytics`.`events` (`id`, `label`)\nSELECT 1, 'first'\nUNION ALL\nSELECT 2, 'second'"
|
||||
);
|
||||
|
||||
let batches = generate_insert_typed_sql_batches(
|
||||
&[String::from("id"), String::from("label")],
|
||||
&[Some(String::from("int")), Some(String::from("string"))],
|
||||
&[vec![json!(1), json!("first")], vec![json!(2), json!("second")]],
|
||||
"events",
|
||||
"analytics",
|
||||
&DatabaseType::Transwarp,
|
||||
None,
|
||||
SqlBatchLimits { max_rows: 100, target_sql_bytes: 76, hard_sql_bytes: Some(76) },
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(batches.iter().map(|(_, rows)| *rows).sum::<usize>(), 2);
|
||||
assert!(batches.iter().all(|(sql, _)| sql.len() <= 76 && sql.contains("\nSELECT ")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inceptor_native_complex_copy_uses_server_side_projection() {
|
||||
assert!(is_transwarp_complex_type("ARRAY<INT>"));
|
||||
assert!(is_transwarp_complex_type("map<string,int>"));
|
||||
assert!(is_transwarp_complex_type("struct<a:int>"));
|
||||
assert!(!is_transwarp_complex_type("STRING"));
|
||||
let columns = vec![db::ColumnInfo { data_type: "ARRAY<INT>".into(), ..Default::default() }];
|
||||
assert!(!should_use_transwarp_server_side_copy(
|
||||
&DatabaseType::Transwarp,
|
||||
&TransferContent::StructureOnly,
|
||||
&columns,
|
||||
));
|
||||
assert!(should_use_transwarp_server_side_copy(
|
||||
&DatabaseType::Transwarp,
|
||||
&TransferContent::StructureAndData,
|
||||
&columns,
|
||||
));
|
||||
assert!(should_use_transwarp_server_side_copy(&DatabaseType::Transwarp, &TransferContent::DataOnly, &columns,));
|
||||
assert!(!should_use_transwarp_server_side_copy(
|
||||
&DatabaseType::Hive,
|
||||
&TransferContent::StructureAndData,
|
||||
&columns,
|
||||
));
|
||||
let sql = transwarp_server_side_copy_sql(
|
||||
&["id".into(), "numbers".into(), "tags".into()],
|
||||
&["id".into(), "numbers".into(), "tags".into()],
|
||||
"events",
|
||||
"source_db",
|
||||
"events",
|
||||
"target_db",
|
||||
true,
|
||||
);
|
||||
assert_eq!(sql, "INSERT INTO `target_db`.`events` (`id`, `numbers`, `tags`) SELECT `id`, `numbers`, `tags` FROM `source_db`.`events`");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn postgres_insert_can_skip_duplicate_rows() {
|
||||
let batches = generate_insert_typed_sql_batches_from_value_rows_with_options(
|
||||
|
||||
@@ -2899,7 +2899,7 @@ async fn execute_sql_statement_with_options_typed_inner(
|
||||
|| crate::sql::has_executable_sql(sql),
|
||||
|db_type| crate::sql::has_executable_sql_for_database(sql, db_type),
|
||||
);
|
||||
if !has_executable_sql {
|
||||
if !has_executable_sql && options.result_session_id.is_none() {
|
||||
return Ok(empty_query_result(0));
|
||||
}
|
||||
|
||||
|
||||
@@ -1520,6 +1520,7 @@ mod tests {
|
||||
table_comment: None,
|
||||
original_table_comment: None,
|
||||
mysql_engine: None,
|
||||
transwarp_create: None,
|
||||
partitioned: false,
|
||||
is_gaussdb_m_mode: false,
|
||||
table_collation: None,
|
||||
|
||||
@@ -52,6 +52,7 @@ fn change_options(table_name: &str, columns: Vec<EditableStructureColumn>) -> Ta
|
||||
table_comment: None,
|
||||
original_table_comment: None,
|
||||
mysql_engine: None,
|
||||
transwarp_create: None,
|
||||
partitioned: false,
|
||||
is_gaussdb_m_mode: false,
|
||||
table_collation: None,
|
||||
|
||||
@@ -48,6 +48,7 @@ fn change_options(table_name: &str, columns: Vec<EditableStructureColumn>) -> Ta
|
||||
table_comment: None,
|
||||
original_table_comment: None,
|
||||
mysql_engine: None,
|
||||
transwarp_create: None,
|
||||
partitioned: false,
|
||||
is_gaussdb_m_mode: false,
|
||||
table_collation: None,
|
||||
|
||||
@@ -62,6 +62,7 @@ async fn live_postgres_concurrent_index_builds_valid_index() {
|
||||
table_comment: None,
|
||||
original_table_comment: None,
|
||||
mysql_engine: None,
|
||||
transwarp_create: None,
|
||||
partitioned: false,
|
||||
is_gaussdb_m_mode: false,
|
||||
table_collation: None,
|
||||
@@ -195,6 +196,7 @@ async fn live_postgres_partitioned_parent_concurrent_request_rejected() {
|
||||
table_comment: None,
|
||||
original_table_comment: None,
|
||||
mysql_engine: None,
|
||||
transwarp_create: None,
|
||||
partitioned: true,
|
||||
is_gaussdb_m_mode: false,
|
||||
table_collation: None,
|
||||
|
||||
@@ -314,6 +314,7 @@ async fn live_postgres_create_partitioned_table_executes() {
|
||||
table_comment: None,
|
||||
original_table_comment: None,
|
||||
mysql_engine: None,
|
||||
transwarp_create: None,
|
||||
partitioned: false,
|
||||
is_gaussdb_m_mode: false,
|
||||
table_collation: None,
|
||||
|
||||
@@ -1017,6 +1017,7 @@ async fn live_sqlserver_table_structure_default_changes_drop_existing_constraint
|
||||
table_comment: None,
|
||||
original_table_comment: None,
|
||||
mysql_engine: None,
|
||||
transwarp_create: None,
|
||||
partitioned: false,
|
||||
is_gaussdb_m_mode: false,
|
||||
table_collation: None,
|
||||
|
||||
@@ -73,6 +73,15 @@ pub fn label_for_key(agent_key: &str) -> Option<&'static str> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn transwarp_products_share_one_driver_package() {
|
||||
for profile in [None, Some("transwarp-inceptor")] {
|
||||
assert_eq!(agent_key(&DatabaseType::Transwarp, profile), Some("transwarp"));
|
||||
}
|
||||
assert_eq!(driver_store_entries().filter(|(key, _)| *key == "transwarp").count(), 1);
|
||||
assert!(!driver_store_entries().any(|(key, _)| key.starts_with("transwarp-")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn h2_profiles_share_the_same_agent() {
|
||||
assert_eq!(agent_key(&DatabaseType::H2, None), Some("h2"));
|
||||
|
||||
@@ -128,6 +128,14 @@ pub fn agent_connect_params_with_role(
|
||||
sap_hana_jdbc_connection_string(config, host, port, database)
|
||||
} else if matches!(config.db_type, DatabaseType::Trino | DatabaseType::PrestoSql) {
|
||||
trino_like_jdbc_connection_string(config, host, port, database)?
|
||||
} else if config.db_type == DatabaseType::Transwarp {
|
||||
match config.connection_string.as_deref().filter(|value| !value.trim().is_empty()) {
|
||||
Some(url) if host != config.host || port != config.port => {
|
||||
crate::models::connection::rewrite_jdbc_url_host(url, host, port)?
|
||||
}
|
||||
Some(url) => url.to_string(),
|
||||
None => String::new(),
|
||||
}
|
||||
} else if config.db_type == DatabaseType::H2 {
|
||||
h2_agent_jdbc_connection_string(config)
|
||||
} else {
|
||||
@@ -1409,6 +1417,22 @@ mod tests {
|
||||
assert_eq!(params["jdbc_driver_class"], "com.example.CustomDriver");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transwarp_jdbc_url_follows_resolved_tunnel_endpoint() {
|
||||
let mut cfg = config(DatabaseType::Transwarp, Some("analytics"));
|
||||
cfg.host = "quark.example.com".to_string();
|
||||
cfg.port = 10000;
|
||||
cfg.connection_string = Some("jdbc:inceptor2://quark.example.com:10000/analytics;fetchSize=500".to_string());
|
||||
cfg.jdbc_driver_class = Some("org.apache.hive.jdbc.HiveDriver".to_string());
|
||||
cfg.jdbc_driver_paths = vec!["C:/drivers/inceptor-sdk.jar".to_string()];
|
||||
|
||||
let params = agent_connect_params(&cfg, "127.0.0.1", 18431, "analytics").unwrap();
|
||||
|
||||
assert_eq!(params["connection_string"], "jdbc:inceptor2://127.0.0.1:18431/analytics;fetchSize=500");
|
||||
assert_eq!(params["jdbc_driver_class"], "org.apache.hive.jdbc.HiveDriver");
|
||||
assert_eq!(params["jdbc_driver_paths"], serde_json::json!(["C:/drivers/inceptor-sdk.jar"]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn db2_agent_connect_params_ignore_driver_registry_ids() {
|
||||
for registry_id in ["db2", "DB2", " db2 ", "db2_zos", "db2i"] {
|
||||
|
||||
@@ -278,6 +278,7 @@ impl SqlDialectProfile {
|
||||
// Oracle for statement splitting so semicolons inside the procedure body are not
|
||||
// misinterpreted as client-side statement terminators.
|
||||
| DatabaseType::Argo
|
||||
| DatabaseType::Transwarp
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -4097,6 +4098,7 @@ BEGIN
|
||||
END;";
|
||||
|
||||
assert_eq!(split_sql_statements_for_database(sql, DatabaseType::Argo), vec![sql.to_string()]);
|
||||
assert_eq!(split_sql_statements_for_database(sql, DatabaseType::Transwarp), vec![sql.to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -340,6 +340,8 @@ pub struct DataGridConditionalUpdateSqlOptions {
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HiveTablePropertiesSqlOptions {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub database_type: Option<DatabaseType>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub schema: Option<String>,
|
||||
pub table_name: String,
|
||||
@@ -1159,6 +1161,15 @@ pub fn build_data_grid_conditional_update_sql(options: DataGridConditionalUpdate
|
||||
}
|
||||
|
||||
pub fn build_hive_table_properties_sql(options: HiveTablePropertiesSqlOptions) -> String {
|
||||
if options.database_type == Some(DatabaseType::Transwarp)
|
||||
&& options.property_name.eq_ignore_ascii_case("transactional")
|
||||
{
|
||||
let database = options.schema.as_deref().unwrap_or("default").replace('\'', "''");
|
||||
let table = options.table_name.replace('\'', "''");
|
||||
return format!(
|
||||
"SELECT transactional FROM system.tables_v WHERE database_name = '{database}' AND table_name = '{table}'"
|
||||
);
|
||||
}
|
||||
let table = qualified_table_name(Some(DatabaseType::Hive), options.schema.as_deref(), &options.table_name);
|
||||
let property = options.property_name.replace('\'', "''");
|
||||
format!("SHOW TBLPROPERTIES {table} ('{property}')")
|
||||
@@ -1755,7 +1766,11 @@ fn build_data_grid_save_statements(
|
||||
.join(", ");
|
||||
statements.push(data_grid_statement(
|
||||
options.database_type,
|
||||
format!("INSERT INTO {table} ({columns}) VALUES ({values})"),
|
||||
if options.database_type == Some(DatabaseType::Transwarp) {
|
||||
format!("INSERT INTO {table} ({columns}) SELECT {values}")
|
||||
} else {
|
||||
format!("INSERT INTO {table} ({columns}) VALUES ({values})")
|
||||
},
|
||||
));
|
||||
}
|
||||
|
||||
@@ -1875,7 +1890,11 @@ fn build_data_grid_rollback_statements(
|
||||
} else {
|
||||
statements.push(data_grid_statement(
|
||||
options.database_type,
|
||||
format!("INSERT INTO {table} ({columns}) VALUES ({values})"),
|
||||
if options.database_type == Some(DatabaseType::Transwarp) {
|
||||
format!("INSERT INTO {table} ({columns}) SELECT {values}")
|
||||
} else {
|
||||
format!("INSERT INTO {table} ({columns}) VALUES ({values})")
|
||||
},
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -6435,6 +6454,7 @@ mod tests {
|
||||
fn builds_hive_table_properties_sql() {
|
||||
assert_eq!(
|
||||
build_hive_table_properties_sql(HiveTablePropertiesSqlOptions {
|
||||
database_type: None,
|
||||
schema: Some("default".to_string()),
|
||||
table_name: "events".to_string(),
|
||||
property_name: "transactional".to_string(),
|
||||
@@ -6572,6 +6592,53 @@ mod tests {
|
||||
assert_eq!(effective_columns(&postgres_options), vec![Some("events.id".to_string())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transwarp_grid_insert_and_restore_use_select_syntax() {
|
||||
let options = DataGridSaveStatementOptions {
|
||||
database_type: Some(DatabaseType::Transwarp),
|
||||
identifier_quote: None,
|
||||
server_version: None,
|
||||
table_meta: DataGridTableMeta {
|
||||
catalog: None,
|
||||
database: None,
|
||||
schema: Some("default".to_string()),
|
||||
table_name: "events".to_string(),
|
||||
primary_keys: vec![],
|
||||
columns: Some(vec![column("id", "int", true, None), column("name", "string", true, None)]),
|
||||
},
|
||||
columns: vec!["id".to_string(), "name".to_string()],
|
||||
source_columns: None,
|
||||
rows: vec![vec![json!(1), json!("before")]],
|
||||
dirty_rows: vec![],
|
||||
deleted_rows: vec![0],
|
||||
new_rows: vec![vec![json!(2), json!("after")]],
|
||||
include_database_name: false,
|
||||
};
|
||||
let result = prepare_data_grid_save(options);
|
||||
assert!(result
|
||||
.statements
|
||||
.iter()
|
||||
.any(|sql| sql.contains("INSERT INTO `default`.`events` (`id`, `name`) SELECT 2, 'after'")));
|
||||
assert!(result
|
||||
.rollback_statements
|
||||
.iter()
|
||||
.any(|sql| sql.contains("INSERT INTO `default`.`events` (`id`, `name`) SELECT 1, 'before'")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transwarp_transactional_property_uses_system_table() {
|
||||
let sql = build_hive_table_properties_sql(HiveTablePropertiesSqlOptions {
|
||||
database_type: Some(DatabaseType::Transwarp),
|
||||
schema: Some("analytics".to_string()),
|
||||
table_name: "events".to_string(),
|
||||
property_name: "transactional".to_string(),
|
||||
});
|
||||
assert_eq!(
|
||||
sql,
|
||||
"SELECT transactional FROM system.tables_v WHERE database_name = 'analytics' AND table_name = 'events'"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn formats_temporal_copy_literals() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -56,6 +56,7 @@ pub fn is_schema_aware(database_type: DatabaseType) -> bool {
|
||||
| DatabaseType::Kyuubi
|
||||
| DatabaseType::Impala
|
||||
| DatabaseType::Argo
|
||||
| DatabaseType::Transwarp
|
||||
| DatabaseType::Spark
|
||||
| DatabaseType::Db2
|
||||
| DatabaseType::Informix
|
||||
|
||||
@@ -894,8 +894,8 @@ pub fn profile_for(db_type: DatabaseType) -> DdlDialectProfile {
|
||||
profile
|
||||
}
|
||||
|
||||
DuckDb | Questdb | Snowflake | Trino | PrestoSql | Hive | Kyuubi | Impala | Argo | Spark | Bigquery
|
||||
| Spanner | Kylin | Ignite | Ignite3 | Oscar | Tdengine | Iotdb | Databricks | Jdbc => {
|
||||
DuckDb | Questdb | Snowflake | Trino | PrestoSql | Hive | Kyuubi | Impala | Argo | Transwarp | Spark
|
||||
| Bigquery | Spanner | Kylin | Ignite | Ignite3 | Oscar | Tdengine | Iotdb | Databricks | Jdbc => {
|
||||
conservative_ansi(db_type)
|
||||
}
|
||||
|
||||
|
||||
@@ -152,6 +152,7 @@ pub fn quote_table_identifier(database_type: Option<DatabaseType>, name: &str) -
|
||||
| DatabaseType::Kyuubi
|
||||
| DatabaseType::Impala
|
||||
| DatabaseType::Argo
|
||||
| DatabaseType::Transwarp
|
||||
| DatabaseType::Spark
|
||||
| DatabaseType::Databricks
|
||||
| DatabaseType::Databend
|
||||
@@ -428,6 +429,7 @@ pub fn quote_transfer_identifier(name: &str, database_type: &DatabaseType) -> St
|
||||
| DatabaseType::Kyuubi
|
||||
| DatabaseType::Impala
|
||||
| DatabaseType::Argo
|
||||
| DatabaseType::Transwarp
|
||||
| DatabaseType::Spark
|
||||
| DatabaseType::Questdb => format!("`{}`", name.replace('`', "``")),
|
||||
DatabaseType::SqlServer => format!("[{}]", name.replace(']', "]]")),
|
||||
|
||||
@@ -782,7 +782,13 @@ pub(super) fn build_select_columns(
|
||||
// natively and users can narrow the projection by editing the SQL.
|
||||
if !matches!(
|
||||
database_type,
|
||||
Some(DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala | DatabaseType::Argo)
|
||||
Some(
|
||||
DatabaseType::Hive
|
||||
| DatabaseType::Kyuubi
|
||||
| DatabaseType::Impala
|
||||
| DatabaseType::Argo
|
||||
| DatabaseType::Transwarp
|
||||
)
|
||||
) {
|
||||
return "*".to_string();
|
||||
}
|
||||
|
||||
@@ -279,6 +279,12 @@ fn build_create_database_statement(options: &CreateDatabaseSqlOptions) -> Result
|
||||
if !supports_create_database_target(options.database_type, options.driver_profile.as_deref()) {
|
||||
return Err(format!("Creating databases is not supported for {}.", database_label(options.database_type)));
|
||||
}
|
||||
if options.database_type == Some(DatabaseType::Transwarp) {
|
||||
return Ok(format!(
|
||||
"CREATE DATABASE IF NOT EXISTS {};",
|
||||
quote_table_identifier(options.database_type, &options.name)
|
||||
));
|
||||
}
|
||||
if is_informix_family(options.database_type, options.driver_profile.as_deref()) {
|
||||
// Informix / GBase 8s accept only a bare `CREATE DATABASE <name>`. The new database
|
||||
// inherits the instance default locale, and the MySQL `CHARACTER SET`/`COLLATE`
|
||||
@@ -347,6 +353,7 @@ pub fn supports_create_database_target(database_type: Option<DatabaseType>, driv
|
||||
| DatabaseType::Highgo
|
||||
| DatabaseType::Kingbase
|
||||
| DatabaseType::Yashandb
|
||||
| DatabaseType::Transwarp
|
||||
)
|
||||
)
|
||||
}
|
||||
@@ -691,7 +698,11 @@ fn supports_truncate_table_cascade(database_type: Option<DatabaseType>) -> bool
|
||||
}
|
||||
|
||||
pub fn build_drop_database_sql(options: DatabaseNameSqlOptions) -> String {
|
||||
format!("DROP DATABASE {};", quote_table_identifier(options.database_type, &options.name))
|
||||
let name = quote_table_identifier(options.database_type, &options.name);
|
||||
if options.database_type == Some(DatabaseType::Transwarp) {
|
||||
return format!("DROP DATABASE IF EXISTS {name};");
|
||||
}
|
||||
format!("DROP DATABASE {name};")
|
||||
}
|
||||
|
||||
pub fn build_update_database_properties_sql(options: DatabasePropertyEditSqlOptions) -> Result<String, String> {
|
||||
@@ -921,6 +932,9 @@ pub fn supports_object_rename(database_type: Option<DatabaseType>, object_type:
|
||||
if database_type == DatabaseType::SqlServer {
|
||||
return true;
|
||||
}
|
||||
if database_type == DatabaseType::Transwarp {
|
||||
return object_type == DatabaseObjectType::Table;
|
||||
}
|
||||
if matches!(object_type, DatabaseObjectType::Procedure | DatabaseObjectType::Function) {
|
||||
return false;
|
||||
}
|
||||
@@ -964,6 +978,14 @@ pub fn build_rename_object_sql(options: RenameObjectSqlOptions) -> Result<String
|
||||
));
|
||||
}
|
||||
|
||||
if database_type == Some(DatabaseType::Transwarp) {
|
||||
return Ok(format!(
|
||||
"ALTER TABLE {} RENAME TO {};",
|
||||
qualified_name(database_type, options.schema.as_deref(), &options.old_name),
|
||||
qualified_name(database_type, options.schema.as_deref(), &options.new_name)
|
||||
));
|
||||
}
|
||||
|
||||
if matches!(database_type, Some(DatabaseType::Mysql | DatabaseType::Goldendb)) {
|
||||
return Ok(format!(
|
||||
"RENAME TABLE {} TO {};",
|
||||
@@ -1539,6 +1561,50 @@ mod tests {
|
||||
assert!(!supports_create_database_target(Some(DatabaseType::Gbase), Some("gbase8a")));
|
||||
assert!(!supports_create_database_target(Some(DatabaseType::Gbase), None));
|
||||
assert!(supports_create_database_target(Some(DatabaseType::Informix), None));
|
||||
assert!(supports_create_database_target(Some(DatabaseType::Transwarp), Some("transwarp-inceptor")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transwarp_database_actions_match_waterdrop_ddl() {
|
||||
assert!(!supports_create_schema_target(Some(DatabaseType::Transwarp)));
|
||||
assert_eq!(
|
||||
build_create_database_sql(CreateDatabaseSqlOptions {
|
||||
database_type: Some(DatabaseType::Transwarp),
|
||||
driver_profile: Some("transwarp-inceptor".to_string()),
|
||||
target: None,
|
||||
parent: None,
|
||||
name: "analytics db".to_string(),
|
||||
charset: None,
|
||||
collation: None,
|
||||
})
|
||||
.unwrap(),
|
||||
"CREATE DATABASE IF NOT EXISTS `analytics db`;"
|
||||
);
|
||||
assert_eq!(
|
||||
build_drop_database_sql(DatabaseNameSqlOptions {
|
||||
database_type: Some(DatabaseType::Transwarp),
|
||||
name: "analytics db".to_string(),
|
||||
}),
|
||||
"DROP DATABASE IF EXISTS `analytics db`;"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transwarp_table_rename_matches_waterdrop_ddl() {
|
||||
assert!(supports_object_rename(Some(DatabaseType::Transwarp), DatabaseObjectType::Table));
|
||||
assert!(!supports_object_rename(Some(DatabaseType::Transwarp), DatabaseObjectType::View));
|
||||
assert!(!supports_database_rename(Some(DatabaseType::Transwarp)));
|
||||
assert_eq!(
|
||||
build_rename_object_sql(RenameObjectSqlOptions {
|
||||
database_type: Some(DatabaseType::Transwarp),
|
||||
object_type: DatabaseObjectType::Table,
|
||||
schema: Some("analytics".to_string()),
|
||||
old_name: "old table".to_string(),
|
||||
new_name: "new table".to_string(),
|
||||
})
|
||||
.unwrap(),
|
||||
"ALTER TABLE `analytics`.`old table` RENAME TO `analytics`.`new table`;"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2938,6 +3004,7 @@ mod tests {
|
||||
table_comment: None,
|
||||
original_table_comment: None,
|
||||
mysql_engine: None,
|
||||
transwarp_create: None,
|
||||
partitioned: false,
|
||||
is_gaussdb_m_mode: false,
|
||||
table_collation: None,
|
||||
|
||||
@@ -9,6 +9,7 @@ mod indexes;
|
||||
mod mysql_engine;
|
||||
mod owner;
|
||||
mod partitions;
|
||||
mod transwarp;
|
||||
mod triggers;
|
||||
mod types;
|
||||
mod util;
|
||||
|
||||
@@ -372,6 +372,15 @@ pub(super) fn build_mysql_existing_column_clause(column: &EditableStructureColum
|
||||
format!("{operation}{position_clause}")
|
||||
}
|
||||
|
||||
pub(super) fn build_transwarp_existing_column_clause(column: &EditableStructureColumn) -> String {
|
||||
let original_name = column.original.as_ref().map(|original| original.name.as_str()).unwrap_or(&column.name);
|
||||
format!(
|
||||
"CHANGE {} {}",
|
||||
quote_ident(StructureDialect::Mysql, original_name),
|
||||
column_definition(StructureDialect::Mysql, column)
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn build_doris_existing_column_sql(
|
||||
table: &str,
|
||||
column: &EditableStructureColumn,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use super::column_alter::build_transwarp_existing_column_clause;
|
||||
use super::column_alter::{
|
||||
build_clickhouse_existing_column_sql, build_dameng_existing_column_sql, build_doris_existing_column_sql,
|
||||
build_duckdb_existing_column_sql, build_h2_existing_column_sql, build_informix_existing_column_sql,
|
||||
@@ -115,7 +116,8 @@ pub(super) fn build_column_sql(options: &TableStructureSqlOptions, warnings: &mu
|
||||
}
|
||||
|
||||
let active_index = active_columns.iter().position(|active| active.id == column.id).unwrap_or(0);
|
||||
let position_clause = if has_original_column_positions {
|
||||
let position_clause = if has_original_column_positions && options.database_type != Some(DatabaseType::Transwarp)
|
||||
{
|
||||
column_position_clause(dialect, &active_columns, active_index)
|
||||
} else {
|
||||
String::new()
|
||||
@@ -248,10 +250,12 @@ pub(super) fn build_column_sql(options: &TableStructureSqlOptions, warnings: &mu
|
||||
} else {
|
||||
column
|
||||
};
|
||||
let clause = build_mysql_existing_column_clause(
|
||||
effective_column,
|
||||
if has_position_change { &position_clause } else { "" },
|
||||
);
|
||||
let position = if has_position_change { &position_clause } else { "" };
|
||||
let clause = if options.database_type == Some(DatabaseType::Transwarp) {
|
||||
build_transwarp_existing_column_clause(effective_column)
|
||||
} else {
|
||||
build_mysql_existing_column_clause(effective_column, position)
|
||||
};
|
||||
if mysql_coalesced_primary_key_change
|
||||
.is_some_and(|change| mysql_auto_increment_touches_primary_key(column, change))
|
||||
{
|
||||
@@ -592,7 +596,9 @@ pub(super) fn build_add_column_sql(
|
||||
driver_profile: Option<&str>,
|
||||
) -> Vec<String> {
|
||||
let definition = column_definition(dialect, column);
|
||||
let mut statements = if is_oracle_like(dialect) || dialect == StructureDialect::Informix {
|
||||
let mut statements = if database_type == Some(DatabaseType::Transwarp) {
|
||||
vec![format!("ALTER TABLE {table} ADD COLUMNS ({definition});")]
|
||||
} else if is_oracle_like(dialect) || dialect == StructureDialect::Informix {
|
||||
vec![format!("ALTER TABLE {table} ADD ({definition});")]
|
||||
} else {
|
||||
let add_keyword = if dialect == StructureDialect::SqlServer
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use super::dialect::{capabilities_for, database_label, dialect_label, StructureDialect};
|
||||
use super::types::TableStructureSqlOptions;
|
||||
use super::util::{clean, qualified_table, quote_string};
|
||||
use crate::models::connection::DatabaseType;
|
||||
|
||||
pub(super) fn build_table_comment_sql(options: &TableStructureSqlOptions, warnings: &mut Vec<String>) -> Vec<String> {
|
||||
let capabilities = capabilities_for(options.database_type, options.driver_profile.as_deref());
|
||||
@@ -19,6 +20,9 @@ pub(super) fn build_table_comment_sql(options: &TableStructureSqlOptions, warnin
|
||||
let dialect = capabilities.dialect;
|
||||
let table = qualified_table(dialect, options.schema.as_deref(), &options.table_name);
|
||||
let quoted = quote_string(&clean(new_comment));
|
||||
if options.database_type == Some(DatabaseType::Transwarp) {
|
||||
return vec![format!("ALTER TABLE {table} SET TBLPROPERTIES ('comment' = {quoted});")];
|
||||
}
|
||||
match dialect {
|
||||
StructureDialect::Mysql | StructureDialect::GaussdbM => {
|
||||
vec![format!("ALTER TABLE {table} COMMENT = {quoted};")]
|
||||
|
||||
@@ -8,6 +8,7 @@ use super::dialect::{capabilities_for, database_label, StructureDialect};
|
||||
use super::foreign_keys::build_foreign_key_sql_for_new_table;
|
||||
use super::indexes::build_create_index_statements;
|
||||
use super::mysql_engine::{append_mysql_table_option, validate_mysql_engine};
|
||||
use super::transwarp;
|
||||
use super::triggers::build_trigger_sql_for_new_table;
|
||||
use super::types::{TableStructureSqlOptions, TableStructureSqlResult};
|
||||
use super::util::{
|
||||
@@ -63,6 +64,7 @@ pub(super) fn build_create_table_sql_with_partition_clause(
|
||||
}
|
||||
validate_columns(&active_columns, &mut warnings);
|
||||
validate_dameng_identity(&options, &active_columns, &mut warnings);
|
||||
transwarp::validate_create_options(&options, &active_columns, &mut warnings);
|
||||
if !warnings.is_empty() {
|
||||
return TableStructureSqlResult { statements: Vec::new(), warnings };
|
||||
}
|
||||
@@ -105,6 +107,9 @@ pub(super) fn build_create_table_sql_with_partition_clause(
|
||||
let mut column_definitions = Vec::new();
|
||||
|
||||
for column in &active_columns {
|
||||
if transwarp::is_partition_column(&options, column) {
|
||||
continue;
|
||||
}
|
||||
let mut data_type = column_data_type(dialect, column);
|
||||
// SQLite accepts AUTOINCREMENT only on an exact INTEGER PRIMARY KEY,
|
||||
// so integer-family aliases are normalized when auto-increment is on.
|
||||
@@ -190,7 +195,8 @@ pub(super) fn build_create_table_sql_with_partition_clause(
|
||||
column_definitions.push(format!("PRIMARY KEY ({pk_list})"));
|
||||
}
|
||||
|
||||
let create_table = format!("CREATE TABLE {table} (\n {}\n)", column_definitions.join(",\n "));
|
||||
let mut create_table = format!("CREATE TABLE {table} (\n {}\n)", column_definitions.join(",\n "));
|
||||
create_table.push_str(&transwarp::create_table_suffix(&options, &active_columns, dialect));
|
||||
let create_table = match partition_clause {
|
||||
Some(clause) => format!("{create_table} {clause}"),
|
||||
None => create_table,
|
||||
@@ -205,7 +211,7 @@ pub(super) fn build_create_table_sql_with_partition_clause(
|
||||
|
||||
if capabilities.comment {
|
||||
let table_comment = clean(options.table_comment.as_deref().unwrap_or(""));
|
||||
if !table_comment.is_empty() {
|
||||
if !table_comment.is_empty() && options.database_type != Some(DatabaseType::Transwarp) {
|
||||
if matches!(dialect, StructureDialect::Mysql | StructureDialect::GaussdbM) {
|
||||
if let Some(last) = statements.last_mut() {
|
||||
append_mysql_table_option(last, &format!("COMMENT = {}", quote_string(&table_comment)));
|
||||
|
||||
@@ -379,6 +379,15 @@ pub(super) fn capabilities_for(
|
||||
rebuild_index: true,
|
||||
..base
|
||||
},
|
||||
// Inceptor accepts ADD COLUMNS and CHANGE, but not DROP COLUMN.
|
||||
Some(DatabaseType::Transwarp) => TableStructureCapabilities {
|
||||
dialect: StructureDialect::Mysql,
|
||||
add_column: true,
|
||||
rename_column: true,
|
||||
alter_existing_column: true,
|
||||
comment: true,
|
||||
..base
|
||||
},
|
||||
_ => base,
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,124 @@
|
||||
use super::column_format::column_data_type;
|
||||
use super::dialect::StructureDialect;
|
||||
use super::types::{EditableStructureColumn, TableStructureSqlOptions, TranswarpCreateTableOptions};
|
||||
use super::util::{clean, quote_new_ident, quote_string};
|
||||
use crate::models::connection::DatabaseType;
|
||||
|
||||
pub(super) fn validate_create_options(
|
||||
options: &TableStructureSqlOptions,
|
||||
active_columns: &[&EditableStructureColumn],
|
||||
warnings: &mut Vec<String>,
|
||||
) {
|
||||
let Some(create) = options.transwarp_create.as_ref() else { return };
|
||||
if options.database_type != Some(DatabaseType::Transwarp) {
|
||||
warnings.push("Inceptor table options require a Transwarp connection.".to_string());
|
||||
return;
|
||||
}
|
||||
for name in create.partition_columns.iter().chain(&create.bucket_columns) {
|
||||
if !active_columns.iter().any(|column| column.name.eq_ignore_ascii_case(name.trim())) {
|
||||
warnings.push(format!("Inceptor column {name:?} does not exist in the table draft."));
|
||||
}
|
||||
}
|
||||
if create.partition_columns.len() >= active_columns.len() && !create.partition_columns.is_empty() {
|
||||
warnings.push("Inceptor partitioned tables need at least one non-partition column.".to_string());
|
||||
}
|
||||
for names in [&create.partition_columns, &create.bucket_columns] {
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
for name in names {
|
||||
if name.trim().is_empty() || !seen.insert(name.trim().to_ascii_lowercase()) {
|
||||
warnings.push("Inceptor partition and bucket columns must be distinct and non-empty.".to_string());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if create
|
||||
.bucket_columns
|
||||
.iter()
|
||||
.any(|bucket| create.partition_columns.iter().any(|partition| partition.eq_ignore_ascii_case(bucket)))
|
||||
{
|
||||
warnings.push("Inceptor bucket columns cannot also be partition columns.".to_string());
|
||||
}
|
||||
if create.bucket_columns.is_empty() != create.bucket_count.is_none()
|
||||
|| create.bucket_count.is_some_and(|count| count == 0 || count > 4096)
|
||||
{
|
||||
warnings.push("Inceptor bucketing requires columns and a bucket count from 1 to 4096.".to_string());
|
||||
}
|
||||
let storage = create.storage_format.as_deref().unwrap_or("").trim();
|
||||
if !storage.is_empty()
|
||||
&& !["ORC", "PARQUET", "TEXTFILE"].iter().any(|allowed| storage.eq_ignore_ascii_case(allowed))
|
||||
{
|
||||
warnings.push("Unsupported Inceptor storage format.".to_string());
|
||||
}
|
||||
if create.transactional && (!storage.eq_ignore_ascii_case("ORC") || create.bucket_columns.is_empty()) {
|
||||
warnings.push("Inceptor transactional tables require ORC storage and bucket columns.".to_string());
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_partition_column(options: &TableStructureSqlOptions, column: &EditableStructureColumn) -> bool {
|
||||
options.database_type == Some(DatabaseType::Transwarp)
|
||||
&& options.transwarp_create.as_ref().is_some_and(|create| {
|
||||
create.partition_columns.iter().any(|name| column.name.eq_ignore_ascii_case(name.trim()))
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn create_table_suffix(
|
||||
options: &TableStructureSqlOptions,
|
||||
active_columns: &[&EditableStructureColumn],
|
||||
dialect: StructureDialect,
|
||||
) -> String {
|
||||
if options.database_type != Some(DatabaseType::Transwarp) {
|
||||
return String::new();
|
||||
}
|
||||
let mut suffix = String::new();
|
||||
let comment = clean(options.table_comment.as_deref().unwrap_or(""));
|
||||
if !comment.is_empty() {
|
||||
suffix.push_str(&format!(" COMMENT {}", quote_string(&comment)));
|
||||
}
|
||||
if let Some(create) = options.transwarp_create.as_ref() {
|
||||
suffix.push_str(&create_options_clause(create, active_columns, dialect));
|
||||
}
|
||||
suffix
|
||||
}
|
||||
|
||||
fn create_options_clause(
|
||||
create: &TranswarpCreateTableOptions,
|
||||
active_columns: &[&EditableStructureColumn],
|
||||
dialect: StructureDialect,
|
||||
) -> String {
|
||||
let mut clauses = String::new();
|
||||
if !create.partition_columns.is_empty() {
|
||||
let partition_columns = create
|
||||
.partition_columns
|
||||
.iter()
|
||||
.map(|name| {
|
||||
let column = active_columns
|
||||
.iter()
|
||||
.find(|column| column.name.eq_ignore_ascii_case(name.trim()))
|
||||
.expect("validated partition column");
|
||||
format!(
|
||||
"{} {}",
|
||||
quote_new_ident(Some(DatabaseType::Transwarp), dialect, &column.name),
|
||||
column_data_type(dialect, column)
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
clauses.push_str(&format!(" PARTITIONED BY ({partition_columns})"));
|
||||
}
|
||||
if let Some(count) = create.bucket_count {
|
||||
let columns = create
|
||||
.bucket_columns
|
||||
.iter()
|
||||
.map(|name| quote_new_ident(Some(DatabaseType::Transwarp), dialect, name.trim()))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
clauses.push_str(&format!(" CLUSTERED BY ({columns}) INTO {count} BUCKETS"));
|
||||
}
|
||||
if let Some(storage) = create.storage_format.as_deref().map(str::trim).filter(|value| !value.is_empty()) {
|
||||
clauses.push_str(&format!(" STORED AS {}", storage.to_ascii_uppercase()));
|
||||
}
|
||||
if create.transactional {
|
||||
clauses.push_str(" TBLPROPERTIES ('transactional'='true')");
|
||||
}
|
||||
clauses
|
||||
}
|
||||
@@ -237,6 +237,8 @@ pub struct TableStructureSqlOptions {
|
||||
pub original_table_comment: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub mysql_engine: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub transwarp_create: Option<TranswarpCreateTableOptions>,
|
||||
/// MySQL only: the table's current default collation
|
||||
/// (`information_schema.TABLES.TABLE_COLLATION`). A column whose collation
|
||||
/// merely matches it inherits the table default, so its `CHARACTER SET` /
|
||||
@@ -259,6 +261,21 @@ pub struct TableStructureSqlOptions {
|
||||
pub is_gaussdb_m_mode: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TranswarpCreateTableOptions {
|
||||
#[serde(default)]
|
||||
pub partition_columns: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub bucket_columns: Vec<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub bucket_count: Option<u32>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub storage_format: Option<String>,
|
||||
#[serde(default)]
|
||||
pub transactional: bool,
|
||||
}
|
||||
|
||||
/// Options for `build_table_partition_operation_sql`.
|
||||
///
|
||||
/// Partition maintenance is deliberately kept out of `TableStructureSqlOptions`:
|
||||
|
||||
@@ -1178,6 +1178,7 @@ impl ConnectionConfig {
|
||||
DatabaseType::Trino => format!("trino://{host}:{port}{db_part}"),
|
||||
DatabaseType::PrestoSql => format!("prestosql://{host}:{port}{db_part}"),
|
||||
DatabaseType::Hive | DatabaseType::Argo => format!("hive://{host}:{port}{db_part}"),
|
||||
DatabaseType::Transwarp => format!("transwarp://{host}:{port}{db_part}"),
|
||||
DatabaseType::Kyuubi => format!("kyuubi://{host}:{port}{db_part}"),
|
||||
DatabaseType::Impala => format!("impala://{host}:{port}{db_part}"),
|
||||
DatabaseType::Spark => format!("spark://{host}:{port}{db_part}"),
|
||||
@@ -1410,7 +1411,7 @@ impl ConnectionConfig {
|
||||
DatabaseType::PrestoSql => {
|
||||
format!("prestosql://{}:{}@{host}:{port}{db_part}", username, password)
|
||||
}
|
||||
DatabaseType::Hive | DatabaseType::Argo => {
|
||||
DatabaseType::Hive | DatabaseType::Argo | DatabaseType::Transwarp => {
|
||||
format!("hive://{}:{}@{host}:{port}{db_part}", username, password)
|
||||
}
|
||||
DatabaseType::Kyuubi => {
|
||||
|
||||
Reference in new issue
Block a user