feat(kyuubi): add native connection support

This commit is contained in:
zipg authored and GitHub committed 2026-08-15 21:55:37 +08:00
1 parent 60e685d6e6
commit 868502326c
38 files changed
+402 -54

No files matched your search

+13
View File
@@ -196,6 +196,13 @@ const AGENT_CATALOG: &[AgentCatalogEntry] = &[
store_visible: true,
profiles: &[],
},
AgentCatalogEntry {
db_type: DatabaseType::Kyuubi,
key: "hive",
label: "Apache Kyuubi",
store_visible: false,
profiles: &[],
},
AgentCatalogEntry {
db_type: DatabaseType::Impala,
key: "hive",
@@ -410,4 +417,10 @@ mod tests {
assert_eq!(agent_key(&DatabaseType::Impala, None), Some("hive"));
assert_eq!(driver_store_entries().filter(|(key, _)| *key == "hive").count(), 1);
}
#[test]
fn kyuubi_reuses_hive_agent_without_duplicate_store_entry() {
assert_eq!(agent_key(&DatabaseType::Kyuubi, None), Some("hive"));
assert_eq!(driver_store_entries().filter(|(key, _)| *key == "hive").count(), 1);
}
}
+1
View File
@@ -172,6 +172,7 @@ macro_rules! agent_connection_pool_database_type {
| DatabaseType::Snowflake
| DatabaseType::Trino
| DatabaseType::Hive
| DatabaseType::Kyuubi
| DatabaseType::Impala
| DatabaseType::Spark
| DatabaseType::Db2
+32 -14
View File
@@ -489,7 +489,13 @@ pub fn build_empty_table_sql(options: TableAdminSqlOptions) -> String {
match options.database_type {
Some(DatabaseType::ClickHouse) => format!("ALTER TABLE {table} DELETE WHERE 1 = 1;"),
Some(DatabaseType::Bigquery) => format!("DELETE FROM {table} WHERE TRUE;"),
Some(DatabaseType::Cassandra | DatabaseType::Hive | DatabaseType::Kylin | DatabaseType::Questdb) => {
Some(
DatabaseType::Cassandra
| DatabaseType::Hive
| DatabaseType::Kyuubi
| DatabaseType::Kylin
| DatabaseType::Questdb,
) => {
format!("TRUNCATE TABLE {table};")
}
Some(DatabaseType::Iotdb) => format!("DELETE FROM {};", iotdb_timeseries_pattern(&table)),
@@ -645,19 +651,20 @@ pub fn build_duplicate_table_structure_sql(options: DuplicateTableStructureSqlOp
let source = qualified_name(options.database_type, options.schema.as_deref(), &options.source_name);
let target =
qualified_duplicate_target_name(options.database_type, options.schema.as_deref(), &options.target_name);
let structure_sql = if matches!(options.database_type, Some(DatabaseType::Mysql | DatabaseType::Impala)) {
format!("CREATE TABLE {target} LIKE {source};")
} else if options.database_type == Some(DatabaseType::Questdb) {
format!("CREATE TABLE {target} (LIKE {source});")
} else if options.database_type.is_some_and(is_postgres_like_structure_copy) {
format!("CREATE TABLE {target} (LIKE {source} INCLUDING ALL);")
} else if options.database_type == Some(DatabaseType::SqlServer) {
format!("SELECT TOP 0 * INTO {target} FROM {source};")
} else if options.database_type.is_some_and(uses_false_predicate_duplicate_structure) {
format!("CREATE TABLE {target} AS SELECT * FROM {source} WHERE 1=0")
} else {
format!("CREATE TABLE {target} AS SELECT * FROM {source} WHERE 0;")
};
let structure_sql =
if matches!(options.database_type, Some(DatabaseType::Mysql | DatabaseType::Kyuubi | DatabaseType::Impala)) {
format!("CREATE TABLE {target} LIKE {source};")
} else if options.database_type == Some(DatabaseType::Questdb) {
format!("CREATE TABLE {target} (LIKE {source});")
} else if options.database_type.is_some_and(is_postgres_like_structure_copy) {
format!("CREATE TABLE {target} (LIKE {source} INCLUDING ALL);")
} else if options.database_type == Some(DatabaseType::SqlServer) {
format!("SELECT TOP 0 * INTO {target} FROM {source};")
} else if options.database_type.is_some_and(uses_false_predicate_duplicate_structure) {
format!("CREATE TABLE {target} AS SELECT * FROM {source} WHERE 1=0")
} else {
format!("CREATE TABLE {target} AS SELECT * FROM {source} WHERE 0;")
};
let mut comment_sql = Vec::new();
if let Some(database_type) =
@@ -1695,6 +1702,17 @@ mod tests {
}),
"CREATE TABLE `dbx_demo`.`connection_test_copy` LIKE `dbx_demo`.`connection_test`;"
);
assert_eq!(
build_duplicate_table_structure_sql(DuplicateTableStructureSqlOptions {
database_type: Some(DatabaseType::Kyuubi),
schema: Some("dbx_demo".to_string()),
source_name: "orders".to_string(),
target_name: "orders_copy".to_string(),
table_comment: None,
column_comments: vec![],
}),
"CREATE TABLE `dbx_demo`.`orders_copy` LIKE `dbx_demo`.`orders`;"
);
assert_eq!(
build_duplicate_table_structure_sql(DuplicateTableStructureSqlOptions {
database_type: Some(DatabaseType::Postgres),
+33
View File
@@ -564,6 +564,7 @@ pub enum DatabaseType {
#[serde(rename = "prestosql")]
PrestoSql,
Hive,
Kyuubi,
Impala,
Spark,
#[serde(rename = "db2")]
@@ -1142,6 +1143,7 @@ impl ConnectionConfig {
DatabaseType::Trino => format!("trino://{host}:{port}{db_part}"),
DatabaseType::PrestoSql => format!("prestosql://{host}:{port}{db_part}"),
DatabaseType::Hive => format!("hive://{host}:{port}{db_part}"),
DatabaseType::Kyuubi => format!("kyuubi://{host}:{port}{db_part}"),
DatabaseType::Impala => format!("impala://{host}:{port}{db_part}"),
DatabaseType::Spark => format!("spark://{host}:{port}{db_part}"),
DatabaseType::Db2 => format!("db2://{host}:{port}{db_part}"),
@@ -1353,6 +1355,15 @@ impl ConnectionConfig {
DatabaseType::Hive => {
format!("hive://{}:{}@{host}:{port}{db_part}", username, password)
}
DatabaseType::Kyuubi => {
if self.username.is_empty() && self.password.is_empty() {
format!("kyuubi://{host}:{port}{db_part}")
} else if self.password.is_empty() {
format!("kyuubi://{}@{host}:{port}{db_part}", username)
} else {
format!("kyuubi://{}:{}@{host}:{port}{db_part}", username, password)
}
}
DatabaseType::Impala => {
if self.username.is_empty() && self.password.is_empty() {
format!("impala://{host}:{port}{db_part}")
@@ -2627,6 +2638,28 @@ mod tests {
);
}
#[test]
fn kyuubi_database_type_and_connection_urls_are_stable() {
assert_eq!(serde_json::to_string(&DatabaseType::Kyuubi).unwrap(), "\"kyuubi\"");
assert_eq!(serde_json::from_str::<DatabaseType>("\"kyuubi\"").unwrap(), DatabaseType::Kyuubi);
let mut config = mysql_config("dbx", "", Some("analytics"));
config.db_type = DatabaseType::Kyuubi;
config.host = "kyuubi.local".to_string();
config.port = 10009;
assert_eq!(config.connection_url(), "kyuubi://dbx@kyuubi.local:10009/analytics");
config.username.clear();
assert_eq!(config.connection_url(), "kyuubi://kyuubi.local:10009/analytics");
config.username = "dbx".to_string();
config.password = "secret".to_string();
assert_eq!(
config.connection_url_with_host(&config.host, config.port),
"kyuubi://dbx:secret@kyuubi.local:10009/analytics"
);
}
#[test]
fn cloudflare_d1_database_type_and_api_url_are_stable() {
assert_eq!(serde_json::to_string(&DatabaseType::CloudflareD1).unwrap(), "\"cloudflare-d1\"");
@@ -53,6 +53,7 @@ pub fn is_schema_aware(database_type: DatabaseType) -> bool {
| DatabaseType::Trino
| DatabaseType::PrestoSql
| DatabaseType::Hive
| DatabaseType::Kyuubi
| DatabaseType::Impala
| DatabaseType::Spark
| DatabaseType::Db2
@@ -669,8 +669,10 @@ pub fn profile_for(db_type: DatabaseType) -> DdlDialectProfile {
ClickHouse => clickhouse_profile(),
DuckDb | Questdb | SapHana | Teradata | Snowflake | Trino | PrestoSql | Hive | Impala | Spark | Db2
| Informix | Bigquery | Kylin | Oscar | Tdengine | Iotdb | Databricks | Jdbc => conservative_ansi(db_type),
DuckDb | Questdb | SapHana | Teradata | Snowflake | Trino | PrestoSql | Hive | Kyuubi | Impala | Spark
| Db2 | Informix | Bigquery | Kylin | Oscar | Tdengine | Iotdb | Databricks | Jdbc => {
conservative_ansi(db_type)
}
// Non-tabular / not applicable for relational CREATE TABLE
Redis | MongoDb | Elasticsearch | Easysearch | Meilisearch | Qdrant | Milvus | Weaviate | ChromaDb | Neo4j
@@ -129,6 +129,7 @@ pub fn quote_table_identifier(database_type: Option<DatabaseType>, name: &str) -
| DatabaseType::StarRocks
| DatabaseType::ManticoreSearch
| DatabaseType::Hive
| DatabaseType::Kyuubi
| DatabaseType::Impala
| DatabaseType::Spark
| DatabaseType::Databend
@@ -379,6 +380,7 @@ pub(crate) fn quote_transfer_identifier(name: &str, database_type: &DatabaseType
| DatabaseType::Doris
| DatabaseType::StarRocks
| DatabaseType::Hive
| DatabaseType::Kyuubi
| DatabaseType::Impala
| DatabaseType::Spark
| DatabaseType::Questdb => format!("`{}`", name.replace('`', "``")),
@@ -529,7 +529,7 @@ pub(super) fn build_select_columns(
.collect::<Vec<_>>()
.join(", ");
}
if !matches!(database_type, Some(DatabaseType::Hive | DatabaseType::Impala)) {
if !matches!(database_type, Some(DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala)) {
return "*".to_string();
}
columns
+5 -1
View File
@@ -3550,7 +3550,11 @@ fn text_data_type(db_type: &DatabaseType) -> &'static str {
DatabaseType::SqlServer => "NVARCHAR(MAX)",
DatabaseType::Oracle | DatabaseType::OceanbaseOracle | DatabaseType::Dameng => "CLOB",
DatabaseType::ClickHouse => "String",
DatabaseType::Hive | DatabaseType::Trino | DatabaseType::PrestoSql | DatabaseType::Databricks => "STRING",
DatabaseType::Hive
| DatabaseType::Kyuubi
| DatabaseType::Trino
| DatabaseType::PrestoSql
| DatabaseType::Databricks => "STRING",
_ => "TEXT",
}
}
+62 -20
View File
@@ -2655,7 +2655,7 @@ pub fn map_column_type(source_type: &str, _source_db: &DatabaseType, target_db:
// Extract basic type, `bigint unsigned` -> `bigint`
base = base.split(' ').next().unwrap_or(base).trim();
if matches!(target_db, DatabaseType::Hive | DatabaseType::Impala) {
if matches!(target_db, DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala) {
return match base {
"tinyint" => "TINYINT".into(),
"smallint" | "int2" => "SMALLINT".into(),
@@ -2859,7 +2859,8 @@ pub fn generate_create_table_ddl(
line.push(' ');
line.push_str(&default_clause);
}
if !c.is_nullable && !matches!(target_db, DatabaseType::Hive | DatabaseType::Impala) {
if !c.is_nullable && !matches!(target_db, DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala)
{
line.push_str(" NOT NULL");
}
if is_mysql_family {
@@ -2882,7 +2883,7 @@ pub fn generate_create_table_ddl(
}
let mut pks = Vec::with_capacity(columns.iter().filter(|c| c.is_primary_key).count());
if !matches!(target_db, DatabaseType::Hive | DatabaseType::Impala) {
if !matches!(target_db, DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala) {
for c in columns {
if c.is_primary_key {
let qname = quote_identifier(&c.name, target_db);
@@ -3320,7 +3321,7 @@ fn generate_upsert_typed_for_transfer(
fn max_transfer_write_rows(db_type: &DatabaseType, mode: &TransferMode) -> usize {
match (db_type, mode) {
(DatabaseType::SqlServer, TransferMode::Append | TransferMode::Overwrite) => MAX_SQLSERVER_INSERT_ROWS,
(DatabaseType::Hive | DatabaseType::Impala, _) => 500,
(DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala, _) => 500,
(DatabaseType::Oracle, TransferMode::Append | TransferMode::Overwrite) => MAX_ORACLE_INSERT_ALL_ROWS,
(DatabaseType::Oracle, TransferMode::Upsert) => MAX_ORACLE_MERGE_ROWS,
_ => usize::MAX,
@@ -6331,7 +6332,7 @@ where
}
#[derive(Default)]
struct ImpalaTransferCursor {
struct HiveServerTransferCursor {
started: bool,
session_id: Option<String>,
}
@@ -6348,13 +6349,13 @@ fn transfer_cursor_sql(
format!("SELECT {col_list} FROM {full_table}")
}
async fn fetch_impala_transfer_batch(
async fn fetch_hive_server_transfer_batch(
state: &AppState,
pool_key: &str,
request: &TransferRequest,
sql: &str,
batch_size: usize,
cursor: &mut ImpalaTransferCursor,
cursor: &mut HiveServerTransferCursor,
) -> Result<db::QueryResult, String> {
let query_timeout_secs = if cursor.started {
0
@@ -6400,7 +6401,7 @@ async fn fetch_impala_transfer_batch(
Ok(result)
}
async fn close_impala_transfer_cursor(state: &AppState, pool_key: &str, cursor: &mut ImpalaTransferCursor) {
async fn close_hive_server_transfer_cursor(state: &AppState, pool_key: &str, cursor: &mut HiveServerTransferCursor) {
let Some(session_id) = cursor.session_id.take() else {
return;
};
@@ -6776,7 +6777,10 @@ where
}
let target_columns = if (request.mode == TransferMode::Upsert
&& !matches!(target_db_type, DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Impala))
&& !matches!(
target_db_type,
DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala
))
|| matches!(target_db_type, DatabaseType::Postgres | DatabaseType::Dameng)
{
get_columns_for_transfer(
@@ -6796,7 +6800,10 @@ where
// Determine effective mode and PK columns for upsert
let (effective_mode, pk_columns) = if request.mode == TransferMode::Upsert {
if matches!(target_db_type, DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Impala) {
if matches!(
target_db_type,
DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala
) {
log::warn!("[transfer] upsert not supported for {:?}, falling back to append", target_db_type);
(TransferMode::Append, vec![])
} else {
@@ -6823,10 +6830,10 @@ where
let batch_size = if request.batch_size == 0 { 1000 } else { request.batch_size };
let mut offset: u64 = 0;
let mut total_transferred: u64 = 0;
// A single Agent cursor keeps Impala rows in one query execution. Re-running
// LIMIT/OFFSET pages cannot be made stable for tables without a unique key.
let use_impala_cursor = matches!(source_db_type, DatabaseType::Impala);
let impala_transfer_sql = use_impala_cursor.then(|| {
// A single Agent cursor keeps Kyuubi/Impala rows in one query execution.
// Re-running LIMIT/OFFSET pages is unstable for tables without a unique key.
let use_hive_server_cursor = matches!(source_db_type, DatabaseType::Kyuubi | DatabaseType::Impala);
let hive_server_transfer_sql = use_hive_server_cursor.then(|| {
transfer_cursor_sql(
&col_names,
table,
@@ -6835,7 +6842,7 @@ where
request.source_catalog.as_deref(),
)
});
let mut impala_cursor = ImpalaTransferCursor::default();
let mut hive_server_cursor = HiveServerTransferCursor::default();
let transfer_result: Result<(), String> = async {
loop {
@@ -6843,10 +6850,17 @@ where
return Err("Cancelled".to_string());
}
let (result, mysql_spatial_markers) = if let Some(sql) = impala_transfer_sql.as_deref() {
let (result, mysql_spatial_markers) = if let Some(sql) = hive_server_transfer_sql.as_deref() {
(
fetch_impala_transfer_batch(state, source_pool_key, request, sql, batch_size, &mut impala_cursor)
.await?,
fetch_hive_server_transfer_batch(
state,
source_pool_key,
request,
sql,
batch_size,
&mut hive_server_cursor,
)
.await?,
false,
)
} else {
@@ -6928,14 +6942,14 @@ where
terminal: false,
});
if (use_impala_cursor && !has_more) || (!use_impala_cursor && row_count < batch_size) {
if (use_hive_server_cursor && !has_more) || (!use_hive_server_cursor && row_count < batch_size) {
break;
}
}
Ok(())
}
.await;
close_impala_transfer_cursor(state, source_pool_key, &mut impala_cursor).await;
close_hive_server_transfer_cursor(state, source_pool_key, &mut hive_server_cursor).await;
transfer_result?;
if pg_compat_transfer {
@@ -9335,6 +9349,34 @@ mod tests {
);
}
#[test]
fn kyuubi_transfer_uses_spark_sql_compatible_ddl_and_batches() {
let cols = vec![
db::ColumnInfo { is_primary_key: true, is_nullable: false, ..test_column("id", "bigint") },
db::ColumnInfo { is_nullable: false, ..test_column("payload", "jsonb") },
];
let ddl = generate_create_table_ddl(
&cols,
"events",
"public",
"warehouse",
&DatabaseType::Kyuubi,
&DatabaseType::Postgres,
None,
None,
);
assert!(ddl.contains("CREATE TABLE IF NOT EXISTS `warehouse`.`events`"));
assert!(ddl.contains("`id` BIGINT"));
assert!(ddl.contains("`payload` STRING"));
assert!(!ddl.contains("PRIMARY KEY"));
assert!(!ddl.contains("NOT NULL"));
assert_eq!(quote_identifier("user`events", &DatabaseType::Kyuubi), "`user``events`");
assert_eq!(max_transfer_write_rows(&DatabaseType::Kyuubi, &TransferMode::Append), 500);
assert_eq!(max_transfer_write_rows(&DatabaseType::Kyuubi, &TransferMode::Upsert), 500);
}
#[test]
fn impala_transfer_uses_impala_compatible_ddl_and_batches() {
let cols = vec![