feat(kyuubi): add native connection support
This commit is contained in:
38 files changed
+402
-54
No files matched your search
@@ -196,6 +196,13 @@ const AGENT_CATALOG: &[AgentCatalogEntry] = &[
|
||||
store_visible: true,
|
||||
profiles: &[],
|
||||
},
|
||||
AgentCatalogEntry {
|
||||
db_type: DatabaseType::Kyuubi,
|
||||
key: "hive",
|
||||
label: "Apache Kyuubi",
|
||||
store_visible: false,
|
||||
profiles: &[],
|
||||
},
|
||||
AgentCatalogEntry {
|
||||
db_type: DatabaseType::Impala,
|
||||
key: "hive",
|
||||
@@ -410,4 +417,10 @@ mod tests {
|
||||
assert_eq!(agent_key(&DatabaseType::Impala, None), Some("hive"));
|
||||
assert_eq!(driver_store_entries().filter(|(key, _)| *key == "hive").count(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kyuubi_reuses_hive_agent_without_duplicate_store_entry() {
|
||||
assert_eq!(agent_key(&DatabaseType::Kyuubi, None), Some("hive"));
|
||||
assert_eq!(driver_store_entries().filter(|(key, _)| *key == "hive").count(), 1);
|
||||
}
|
||||
}
|
||||
@@ -172,6 +172,7 @@ macro_rules! agent_connection_pool_database_type {
|
||||
| DatabaseType::Snowflake
|
||||
| DatabaseType::Trino
|
||||
| DatabaseType::Hive
|
||||
| DatabaseType::Kyuubi
|
||||
| DatabaseType::Impala
|
||||
| DatabaseType::Spark
|
||||
| DatabaseType::Db2
|
||||
|
||||
@@ -489,7 +489,13 @@ pub fn build_empty_table_sql(options: TableAdminSqlOptions) -> String {
|
||||
match options.database_type {
|
||||
Some(DatabaseType::ClickHouse) => format!("ALTER TABLE {table} DELETE WHERE 1 = 1;"),
|
||||
Some(DatabaseType::Bigquery) => format!("DELETE FROM {table} WHERE TRUE;"),
|
||||
Some(DatabaseType::Cassandra | DatabaseType::Hive | DatabaseType::Kylin | DatabaseType::Questdb) => {
|
||||
Some(
|
||||
DatabaseType::Cassandra
|
||||
| DatabaseType::Hive
|
||||
| DatabaseType::Kyuubi
|
||||
| DatabaseType::Kylin
|
||||
| DatabaseType::Questdb,
|
||||
) => {
|
||||
format!("TRUNCATE TABLE {table};")
|
||||
}
|
||||
Some(DatabaseType::Iotdb) => format!("DELETE FROM {};", iotdb_timeseries_pattern(&table)),
|
||||
@@ -645,19 +651,20 @@ pub fn build_duplicate_table_structure_sql(options: DuplicateTableStructureSqlOp
|
||||
let source = qualified_name(options.database_type, options.schema.as_deref(), &options.source_name);
|
||||
let target =
|
||||
qualified_duplicate_target_name(options.database_type, options.schema.as_deref(), &options.target_name);
|
||||
let structure_sql = if matches!(options.database_type, Some(DatabaseType::Mysql | DatabaseType::Impala)) {
|
||||
format!("CREATE TABLE {target} LIKE {source};")
|
||||
} else if options.database_type == Some(DatabaseType::Questdb) {
|
||||
format!("CREATE TABLE {target} (LIKE {source});")
|
||||
} else if options.database_type.is_some_and(is_postgres_like_structure_copy) {
|
||||
format!("CREATE TABLE {target} (LIKE {source} INCLUDING ALL);")
|
||||
} else if options.database_type == Some(DatabaseType::SqlServer) {
|
||||
format!("SELECT TOP 0 * INTO {target} FROM {source};")
|
||||
} else if options.database_type.is_some_and(uses_false_predicate_duplicate_structure) {
|
||||
format!("CREATE TABLE {target} AS SELECT * FROM {source} WHERE 1=0")
|
||||
} else {
|
||||
format!("CREATE TABLE {target} AS SELECT * FROM {source} WHERE 0;")
|
||||
};
|
||||
let structure_sql =
|
||||
if matches!(options.database_type, Some(DatabaseType::Mysql | DatabaseType::Kyuubi | DatabaseType::Impala)) {
|
||||
format!("CREATE TABLE {target} LIKE {source};")
|
||||
} else if options.database_type == Some(DatabaseType::Questdb) {
|
||||
format!("CREATE TABLE {target} (LIKE {source});")
|
||||
} else if options.database_type.is_some_and(is_postgres_like_structure_copy) {
|
||||
format!("CREATE TABLE {target} (LIKE {source} INCLUDING ALL);")
|
||||
} else if options.database_type == Some(DatabaseType::SqlServer) {
|
||||
format!("SELECT TOP 0 * INTO {target} FROM {source};")
|
||||
} else if options.database_type.is_some_and(uses_false_predicate_duplicate_structure) {
|
||||
format!("CREATE TABLE {target} AS SELECT * FROM {source} WHERE 1=0")
|
||||
} else {
|
||||
format!("CREATE TABLE {target} AS SELECT * FROM {source} WHERE 0;")
|
||||
};
|
||||
|
||||
let mut comment_sql = Vec::new();
|
||||
if let Some(database_type) =
|
||||
@@ -1695,6 +1702,17 @@ mod tests {
|
||||
}),
|
||||
"CREATE TABLE `dbx_demo`.`connection_test_copy` LIKE `dbx_demo`.`connection_test`;"
|
||||
);
|
||||
assert_eq!(
|
||||
build_duplicate_table_structure_sql(DuplicateTableStructureSqlOptions {
|
||||
database_type: Some(DatabaseType::Kyuubi),
|
||||
schema: Some("dbx_demo".to_string()),
|
||||
source_name: "orders".to_string(),
|
||||
target_name: "orders_copy".to_string(),
|
||||
table_comment: None,
|
||||
column_comments: vec![],
|
||||
}),
|
||||
"CREATE TABLE `dbx_demo`.`orders_copy` LIKE `dbx_demo`.`orders`;"
|
||||
);
|
||||
assert_eq!(
|
||||
build_duplicate_table_structure_sql(DuplicateTableStructureSqlOptions {
|
||||
database_type: Some(DatabaseType::Postgres),
|
||||
|
||||
@@ -564,6 +564,7 @@ pub enum DatabaseType {
|
||||
#[serde(rename = "prestosql")]
|
||||
PrestoSql,
|
||||
Hive,
|
||||
Kyuubi,
|
||||
Impala,
|
||||
Spark,
|
||||
#[serde(rename = "db2")]
|
||||
@@ -1142,6 +1143,7 @@ impl ConnectionConfig {
|
||||
DatabaseType::Trino => format!("trino://{host}:{port}{db_part}"),
|
||||
DatabaseType::PrestoSql => format!("prestosql://{host}:{port}{db_part}"),
|
||||
DatabaseType::Hive => format!("hive://{host}:{port}{db_part}"),
|
||||
DatabaseType::Kyuubi => format!("kyuubi://{host}:{port}{db_part}"),
|
||||
DatabaseType::Impala => format!("impala://{host}:{port}{db_part}"),
|
||||
DatabaseType::Spark => format!("spark://{host}:{port}{db_part}"),
|
||||
DatabaseType::Db2 => format!("db2://{host}:{port}{db_part}"),
|
||||
@@ -1353,6 +1355,15 @@ impl ConnectionConfig {
|
||||
DatabaseType::Hive => {
|
||||
format!("hive://{}:{}@{host}:{port}{db_part}", username, password)
|
||||
}
|
||||
DatabaseType::Kyuubi => {
|
||||
if self.username.is_empty() && self.password.is_empty() {
|
||||
format!("kyuubi://{host}:{port}{db_part}")
|
||||
} else if self.password.is_empty() {
|
||||
format!("kyuubi://{}@{host}:{port}{db_part}", username)
|
||||
} else {
|
||||
format!("kyuubi://{}:{}@{host}:{port}{db_part}", username, password)
|
||||
}
|
||||
}
|
||||
DatabaseType::Impala => {
|
||||
if self.username.is_empty() && self.password.is_empty() {
|
||||
format!("impala://{host}:{port}{db_part}")
|
||||
@@ -2627,6 +2638,28 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kyuubi_database_type_and_connection_urls_are_stable() {
|
||||
assert_eq!(serde_json::to_string(&DatabaseType::Kyuubi).unwrap(), "\"kyuubi\"");
|
||||
assert_eq!(serde_json::from_str::<DatabaseType>("\"kyuubi\"").unwrap(), DatabaseType::Kyuubi);
|
||||
|
||||
let mut config = mysql_config("dbx", "", Some("analytics"));
|
||||
config.db_type = DatabaseType::Kyuubi;
|
||||
config.host = "kyuubi.local".to_string();
|
||||
config.port = 10009;
|
||||
assert_eq!(config.connection_url(), "kyuubi://dbx@kyuubi.local:10009/analytics");
|
||||
|
||||
config.username.clear();
|
||||
assert_eq!(config.connection_url(), "kyuubi://kyuubi.local:10009/analytics");
|
||||
|
||||
config.username = "dbx".to_string();
|
||||
config.password = "secret".to_string();
|
||||
assert_eq!(
|
||||
config.connection_url_with_host(&config.host, config.port),
|
||||
"kyuubi://dbx:secret@kyuubi.local:10009/analytics"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cloudflare_d1_database_type_and_api_url_are_stable() {
|
||||
assert_eq!(serde_json::to_string(&DatabaseType::CloudflareD1).unwrap(), "\"cloudflare-d1\"");
|
||||
|
||||
@@ -53,6 +53,7 @@ pub fn is_schema_aware(database_type: DatabaseType) -> bool {
|
||||
| DatabaseType::Trino
|
||||
| DatabaseType::PrestoSql
|
||||
| DatabaseType::Hive
|
||||
| DatabaseType::Kyuubi
|
||||
| DatabaseType::Impala
|
||||
| DatabaseType::Spark
|
||||
| DatabaseType::Db2
|
||||
|
||||
@@ -669,8 +669,10 @@ pub fn profile_for(db_type: DatabaseType) -> DdlDialectProfile {
|
||||
|
||||
ClickHouse => clickhouse_profile(),
|
||||
|
||||
DuckDb | Questdb | SapHana | Teradata | Snowflake | Trino | PrestoSql | Hive | Impala | Spark | Db2
|
||||
| Informix | Bigquery | Kylin | Oscar | Tdengine | Iotdb | Databricks | Jdbc => conservative_ansi(db_type),
|
||||
DuckDb | Questdb | SapHana | Teradata | Snowflake | Trino | PrestoSql | Hive | Kyuubi | Impala | Spark
|
||||
| Db2 | Informix | Bigquery | Kylin | Oscar | Tdengine | Iotdb | Databricks | Jdbc => {
|
||||
conservative_ansi(db_type)
|
||||
}
|
||||
|
||||
// Non-tabular / not applicable for relational CREATE TABLE
|
||||
Redis | MongoDb | Elasticsearch | Easysearch | Meilisearch | Qdrant | Milvus | Weaviate | ChromaDb | Neo4j
|
||||
|
||||
@@ -129,6 +129,7 @@ pub fn quote_table_identifier(database_type: Option<DatabaseType>, name: &str) -
|
||||
| DatabaseType::StarRocks
|
||||
| DatabaseType::ManticoreSearch
|
||||
| DatabaseType::Hive
|
||||
| DatabaseType::Kyuubi
|
||||
| DatabaseType::Impala
|
||||
| DatabaseType::Spark
|
||||
| DatabaseType::Databend
|
||||
@@ -379,6 +380,7 @@ pub(crate) fn quote_transfer_identifier(name: &str, database_type: &DatabaseType
|
||||
| DatabaseType::Doris
|
||||
| DatabaseType::StarRocks
|
||||
| DatabaseType::Hive
|
||||
| DatabaseType::Kyuubi
|
||||
| DatabaseType::Impala
|
||||
| DatabaseType::Spark
|
||||
| DatabaseType::Questdb => format!("`{}`", name.replace('`', "``")),
|
||||
|
||||
@@ -529,7 +529,7 @@ pub(super) fn build_select_columns(
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
}
|
||||
if !matches!(database_type, Some(DatabaseType::Hive | DatabaseType::Impala)) {
|
||||
if !matches!(database_type, Some(DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala)) {
|
||||
return "*".to_string();
|
||||
}
|
||||
columns
|
||||
|
||||
@@ -3550,7 +3550,11 @@ fn text_data_type(db_type: &DatabaseType) -> &'static str {
|
||||
DatabaseType::SqlServer => "NVARCHAR(MAX)",
|
||||
DatabaseType::Oracle | DatabaseType::OceanbaseOracle | DatabaseType::Dameng => "CLOB",
|
||||
DatabaseType::ClickHouse => "String",
|
||||
DatabaseType::Hive | DatabaseType::Trino | DatabaseType::PrestoSql | DatabaseType::Databricks => "STRING",
|
||||
DatabaseType::Hive
|
||||
| DatabaseType::Kyuubi
|
||||
| DatabaseType::Trino
|
||||
| DatabaseType::PrestoSql
|
||||
| DatabaseType::Databricks => "STRING",
|
||||
_ => "TEXT",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2655,7 +2655,7 @@ pub fn map_column_type(source_type: &str, _source_db: &DatabaseType, target_db:
|
||||
// Extract basic type, `bigint unsigned` -> `bigint`
|
||||
base = base.split(' ').next().unwrap_or(base).trim();
|
||||
|
||||
if matches!(target_db, DatabaseType::Hive | DatabaseType::Impala) {
|
||||
if matches!(target_db, DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala) {
|
||||
return match base {
|
||||
"tinyint" => "TINYINT".into(),
|
||||
"smallint" | "int2" => "SMALLINT".into(),
|
||||
@@ -2859,7 +2859,8 @@ pub fn generate_create_table_ddl(
|
||||
line.push(' ');
|
||||
line.push_str(&default_clause);
|
||||
}
|
||||
if !c.is_nullable && !matches!(target_db, DatabaseType::Hive | DatabaseType::Impala) {
|
||||
if !c.is_nullable && !matches!(target_db, DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala)
|
||||
{
|
||||
line.push_str(" NOT NULL");
|
||||
}
|
||||
if is_mysql_family {
|
||||
@@ -2882,7 +2883,7 @@ pub fn generate_create_table_ddl(
|
||||
}
|
||||
|
||||
let mut pks = Vec::with_capacity(columns.iter().filter(|c| c.is_primary_key).count());
|
||||
if !matches!(target_db, DatabaseType::Hive | DatabaseType::Impala) {
|
||||
if !matches!(target_db, DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala) {
|
||||
for c in columns {
|
||||
if c.is_primary_key {
|
||||
let qname = quote_identifier(&c.name, target_db);
|
||||
@@ -3320,7 +3321,7 @@ fn generate_upsert_typed_for_transfer(
|
||||
fn max_transfer_write_rows(db_type: &DatabaseType, mode: &TransferMode) -> usize {
|
||||
match (db_type, mode) {
|
||||
(DatabaseType::SqlServer, TransferMode::Append | TransferMode::Overwrite) => MAX_SQLSERVER_INSERT_ROWS,
|
||||
(DatabaseType::Hive | DatabaseType::Impala, _) => 500,
|
||||
(DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala, _) => 500,
|
||||
(DatabaseType::Oracle, TransferMode::Append | TransferMode::Overwrite) => MAX_ORACLE_INSERT_ALL_ROWS,
|
||||
(DatabaseType::Oracle, TransferMode::Upsert) => MAX_ORACLE_MERGE_ROWS,
|
||||
_ => usize::MAX,
|
||||
@@ -6331,7 +6332,7 @@ where
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ImpalaTransferCursor {
|
||||
struct HiveServerTransferCursor {
|
||||
started: bool,
|
||||
session_id: Option<String>,
|
||||
}
|
||||
@@ -6348,13 +6349,13 @@ fn transfer_cursor_sql(
|
||||
format!("SELECT {col_list} FROM {full_table}")
|
||||
}
|
||||
|
||||
async fn fetch_impala_transfer_batch(
|
||||
async fn fetch_hive_server_transfer_batch(
|
||||
state: &AppState,
|
||||
pool_key: &str,
|
||||
request: &TransferRequest,
|
||||
sql: &str,
|
||||
batch_size: usize,
|
||||
cursor: &mut ImpalaTransferCursor,
|
||||
cursor: &mut HiveServerTransferCursor,
|
||||
) -> Result<db::QueryResult, String> {
|
||||
let query_timeout_secs = if cursor.started {
|
||||
0
|
||||
@@ -6400,7 +6401,7 @@ async fn fetch_impala_transfer_batch(
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn close_impala_transfer_cursor(state: &AppState, pool_key: &str, cursor: &mut ImpalaTransferCursor) {
|
||||
async fn close_hive_server_transfer_cursor(state: &AppState, pool_key: &str, cursor: &mut HiveServerTransferCursor) {
|
||||
let Some(session_id) = cursor.session_id.take() else {
|
||||
return;
|
||||
};
|
||||
@@ -6776,7 +6777,10 @@ where
|
||||
}
|
||||
|
||||
let target_columns = if (request.mode == TransferMode::Upsert
|
||||
&& !matches!(target_db_type, DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Impala))
|
||||
&& !matches!(
|
||||
target_db_type,
|
||||
DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala
|
||||
))
|
||||
|| matches!(target_db_type, DatabaseType::Postgres | DatabaseType::Dameng)
|
||||
{
|
||||
get_columns_for_transfer(
|
||||
@@ -6796,7 +6800,10 @@ where
|
||||
|
||||
// Determine effective mode and PK columns for upsert
|
||||
let (effective_mode, pk_columns) = if request.mode == TransferMode::Upsert {
|
||||
if matches!(target_db_type, DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Impala) {
|
||||
if matches!(
|
||||
target_db_type,
|
||||
DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala
|
||||
) {
|
||||
log::warn!("[transfer] upsert not supported for {:?}, falling back to append", target_db_type);
|
||||
(TransferMode::Append, vec![])
|
||||
} else {
|
||||
@@ -6823,10 +6830,10 @@ where
|
||||
let batch_size = if request.batch_size == 0 { 1000 } else { request.batch_size };
|
||||
let mut offset: u64 = 0;
|
||||
let mut total_transferred: u64 = 0;
|
||||
// A single Agent cursor keeps Impala rows in one query execution. Re-running
|
||||
// LIMIT/OFFSET pages cannot be made stable for tables without a unique key.
|
||||
let use_impala_cursor = matches!(source_db_type, DatabaseType::Impala);
|
||||
let impala_transfer_sql = use_impala_cursor.then(|| {
|
||||
// A single Agent cursor keeps Kyuubi/Impala rows in one query execution.
|
||||
// Re-running LIMIT/OFFSET pages is unstable for tables without a unique key.
|
||||
let use_hive_server_cursor = matches!(source_db_type, DatabaseType::Kyuubi | DatabaseType::Impala);
|
||||
let hive_server_transfer_sql = use_hive_server_cursor.then(|| {
|
||||
transfer_cursor_sql(
|
||||
&col_names,
|
||||
table,
|
||||
@@ -6835,7 +6842,7 @@ where
|
||||
request.source_catalog.as_deref(),
|
||||
)
|
||||
});
|
||||
let mut impala_cursor = ImpalaTransferCursor::default();
|
||||
let mut hive_server_cursor = HiveServerTransferCursor::default();
|
||||
|
||||
let transfer_result: Result<(), String> = async {
|
||||
loop {
|
||||
@@ -6843,10 +6850,17 @@ where
|
||||
return Err("Cancelled".to_string());
|
||||
}
|
||||
|
||||
let (result, mysql_spatial_markers) = if let Some(sql) = impala_transfer_sql.as_deref() {
|
||||
let (result, mysql_spatial_markers) = if let Some(sql) = hive_server_transfer_sql.as_deref() {
|
||||
(
|
||||
fetch_impala_transfer_batch(state, source_pool_key, request, sql, batch_size, &mut impala_cursor)
|
||||
.await?,
|
||||
fetch_hive_server_transfer_batch(
|
||||
state,
|
||||
source_pool_key,
|
||||
request,
|
||||
sql,
|
||||
batch_size,
|
||||
&mut hive_server_cursor,
|
||||
)
|
||||
.await?,
|
||||
false,
|
||||
)
|
||||
} else {
|
||||
@@ -6928,14 +6942,14 @@ where
|
||||
terminal: false,
|
||||
});
|
||||
|
||||
if (use_impala_cursor && !has_more) || (!use_impala_cursor && row_count < batch_size) {
|
||||
if (use_hive_server_cursor && !has_more) || (!use_hive_server_cursor && row_count < batch_size) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
close_impala_transfer_cursor(state, source_pool_key, &mut impala_cursor).await;
|
||||
close_hive_server_transfer_cursor(state, source_pool_key, &mut hive_server_cursor).await;
|
||||
transfer_result?;
|
||||
|
||||
if pg_compat_transfer {
|
||||
@@ -9335,6 +9349,34 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kyuubi_transfer_uses_spark_sql_compatible_ddl_and_batches() {
|
||||
let cols = vec![
|
||||
db::ColumnInfo { is_primary_key: true, is_nullable: false, ..test_column("id", "bigint") },
|
||||
db::ColumnInfo { is_nullable: false, ..test_column("payload", "jsonb") },
|
||||
];
|
||||
|
||||
let ddl = generate_create_table_ddl(
|
||||
&cols,
|
||||
"events",
|
||||
"public",
|
||||
"warehouse",
|
||||
&DatabaseType::Kyuubi,
|
||||
&DatabaseType::Postgres,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
assert!(ddl.contains("CREATE TABLE IF NOT EXISTS `warehouse`.`events`"));
|
||||
assert!(ddl.contains("`id` BIGINT"));
|
||||
assert!(ddl.contains("`payload` STRING"));
|
||||
assert!(!ddl.contains("PRIMARY KEY"));
|
||||
assert!(!ddl.contains("NOT NULL"));
|
||||
assert_eq!(quote_identifier("user`events", &DatabaseType::Kyuubi), "`user``events`");
|
||||
assert_eq!(max_transfer_write_rows(&DatabaseType::Kyuubi, &TransferMode::Append), 500);
|
||||
assert_eq!(max_transfer_write_rows(&DatabaseType::Kyuubi, &TransferMode::Upsert), 500);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn impala_transfer_uses_impala_compatible_ddl_and_batches() {
|
||||
let cols = vec![
|
||||
|
||||
Reference in new issue
Block a user