feat(impala): add full database support

This commit is contained in:
zipg authored and GitHub committed 2026-08-14 20:41:57 +08:00
1 parent 185ff68d78
commit a6d9e6818e
54 files changed
+1092 -148

No files matched your search

+13
View File
@@ -196,6 +196,13 @@ const AGENT_CATALOG: &[AgentCatalogEntry] = &[
store_visible: true,
profiles: &[],
},
AgentCatalogEntry {
db_type: DatabaseType::Impala,
key: "hive",
label: "Apache Impala",
store_visible: false,
profiles: &[],
},
AgentCatalogEntry {
db_type: DatabaseType::Spark,
key: "spark",
@@ -397,4 +404,10 @@ mod tests {
assert_eq!(label_for_key("duckdb"), Some("DuckDB"));
assert!(!is_agent_type(&DatabaseType::DuckDb));
}
#[test]
fn impala_reuses_hive_agent_without_duplicate_store_entry() {
assert_eq!(agent_key(&DatabaseType::Impala, None), Some("hive"));
assert_eq!(driver_store_entries().filter(|(key, _)| *key == "hive").count(), 1);
}
}
+2
View File
@@ -110,6 +110,7 @@ pub fn agent_connect_params_with_role(
"jdbc_driver_class": agent_jdbc_driver_class(config),
"jdbc_driver_paths": &config.jdbc_driver_paths,
"sessionRole": session_role.as_str(),
"database_type": config.db_type,
});
if config.db_type == DatabaseType::ZooKeeper {
params["connection_timeout_ms"] = serde_json::json!(
@@ -700,6 +701,7 @@ mod tests {
fn agent_connect_params_default_to_workload_session_role() {
let params = agent_connect_params(&config(DatabaseType::H2, Some("test")), "127.0.0.1", 9092, "test");
assert_eq!(params["sessionRole"], "workload");
assert_eq!(params["database_type"], "h2");
}
#[test]
+1
View File
@@ -172,6 +172,7 @@ macro_rules! agent_connection_pool_database_type {
| DatabaseType::Snowflake
| DatabaseType::Trino
| DatabaseType::Hive
| DatabaseType::Impala
| DatabaseType::Spark
| DatabaseType::Db2
| DatabaseType::Informix
+12 -1
View File
@@ -645,7 +645,7 @@ pub fn build_duplicate_table_structure_sql(options: DuplicateTableStructureSqlOp
let source = qualified_name(options.database_type, options.schema.as_deref(), &options.source_name);
let target =
qualified_duplicate_target_name(options.database_type, options.schema.as_deref(), &options.target_name);
let structure_sql = if options.database_type == Some(DatabaseType::Mysql) {
let structure_sql = if matches!(options.database_type, Some(DatabaseType::Mysql | DatabaseType::Impala)) {
format!("CREATE TABLE {target} LIKE {source};")
} else if options.database_type == Some(DatabaseType::Questdb) {
format!("CREATE TABLE {target} (LIKE {source});")
@@ -1684,6 +1684,17 @@ mod tests {
}),
"CREATE TABLE \"public\".\"users_copy\" (LIKE \"public\".\"users\" INCLUDING ALL);"
);
assert_eq!(
build_duplicate_table_structure_sql(DuplicateTableStructureSqlOptions {
database_type: Some(DatabaseType::Impala),
schema: Some("dbx_demo".to_string()),
source_name: "connection_test".to_string(),
target_name: "connection_test_copy".to_string(),
table_comment: None,
column_comments: vec![],
}),
"CREATE TABLE `dbx_demo`.`connection_test_copy` LIKE `dbx_demo`.`connection_test`;"
);
assert_eq!(
build_duplicate_table_structure_sql(DuplicateTableStructureSqlOptions {
database_type: Some(DatabaseType::Postgres),
+28
View File
@@ -564,6 +564,7 @@ pub enum DatabaseType {
#[serde(rename = "prestosql")]
PrestoSql,
Hive,
Impala,
Spark,
#[serde(rename = "db2")]
Db2,
@@ -1141,6 +1142,7 @@ impl ConnectionConfig {
DatabaseType::Trino => format!("trino://{host}:{port}{db_part}"),
DatabaseType::PrestoSql => format!("prestosql://{host}:{port}{db_part}"),
DatabaseType::Hive => format!("hive://{host}:{port}{db_part}"),
DatabaseType::Impala => format!("impala://{host}:{port}{db_part}"),
DatabaseType::Spark => format!("spark://{host}:{port}{db_part}"),
DatabaseType::Db2 => format!("db2://{host}:{port}{db_part}"),
DatabaseType::Informix => format!("informix://{host}:{port}{db_part}"),
@@ -1351,6 +1353,13 @@ impl ConnectionConfig {
DatabaseType::Hive => {
format!("hive://{}:{}@{host}:{port}{db_part}", username, password)
}
DatabaseType::Impala => {
if self.username.is_empty() && self.password.is_empty() {
format!("impala://{host}:{port}{db_part}")
} else {
format!("impala://{}:{}@{host}:{port}{db_part}", username, password)
}
}
DatabaseType::Spark => {
format!("spark://{}:{}@{host}:{port}{db_part}", username, password)
}
@@ -2599,6 +2608,25 @@ mod tests {
assert_eq!(serde_json::from_str::<DatabaseType>("\"easysearch\"").unwrap(), DatabaseType::Easysearch);
}
#[test]
fn impala_database_type_and_connection_urls_are_stable() {
assert_eq!(serde_json::to_string(&DatabaseType::Impala).unwrap(), "\"impala\"");
assert_eq!(serde_json::from_str::<DatabaseType>("\"impala\"").unwrap(), DatabaseType::Impala);
let mut config = mysql_config("", "", Some("analytics"));
config.db_type = DatabaseType::Impala;
config.host = "impala.local".to_string();
config.port = 21050;
assert_eq!(config.connection_url(), "impala://impala.local:21050/analytics");
config.username = "analyst".to_string();
config.password = "secret".to_string();
assert_eq!(
config.connection_url_with_host(&config.host, config.port),
"impala://analyst:secret@impala.local:21050/analytics"
);
}
#[test]
fn cloudflare_d1_database_type_and_api_url_are_stable() {
assert_eq!(serde_json::to_string(&DatabaseType::CloudflareD1).unwrap(), "\"cloudflare-d1\"");
@@ -53,6 +53,7 @@ pub fn is_schema_aware(database_type: DatabaseType) -> bool {
| DatabaseType::Trino
| DatabaseType::PrestoSql
| DatabaseType::Hive
| DatabaseType::Impala
| DatabaseType::Spark
| DatabaseType::Db2
| DatabaseType::Informix
@@ -669,8 +669,8 @@ pub fn profile_for(db_type: DatabaseType) -> DdlDialectProfile {
ClickHouse => clickhouse_profile(),
DuckDb | Questdb | SapHana | Teradata | Snowflake | Trino | PrestoSql | Hive | Spark | Db2 | Informix
| Bigquery | Kylin | Oscar | Tdengine | Iotdb | Databricks | Jdbc => conservative_ansi(db_type),
DuckDb | Questdb | SapHana | Teradata | Snowflake | Trino | PrestoSql | Hive | Impala | Spark | Db2
| Informix | Bigquery | Kylin | Oscar | Tdengine | Iotdb | Databricks | Jdbc => conservative_ansi(db_type),
// Non-tabular / not applicable for relational CREATE TABLE
Redis | MongoDb | Elasticsearch | Easysearch | Meilisearch | Qdrant | Milvus | Weaviate | ChromaDb | Neo4j
@@ -129,6 +129,7 @@ pub fn quote_table_identifier(database_type: Option<DatabaseType>, name: &str) -
| DatabaseType::StarRocks
| DatabaseType::ManticoreSearch
| DatabaseType::Hive
| DatabaseType::Impala
| DatabaseType::Spark
| DatabaseType::Databend
| DatabaseType::Tdengine
@@ -378,6 +379,7 @@ pub(crate) fn quote_transfer_identifier(name: &str, database_type: &DatabaseType
| DatabaseType::Doris
| DatabaseType::StarRocks
| DatabaseType::Hive
| DatabaseType::Impala
| DatabaseType::Spark
| DatabaseType::Questdb => format!("`{}`", name.replace('`', "``")),
DatabaseType::SqlServer => format!("[{}]", name.replace(']', "]]")),
@@ -186,6 +186,11 @@ pub fn build_table_data_select_sql(options: TableDataSelectSqlOptions) -> String
let default_order_by = if database_type == Some(DatabaseType::InfluxDb) {
// InfluxQL only allows sorting of timestamp column
Some("time DESC".to_string())
} else if database_type == Some(DatabaseType::Impala) {
// Impala requires ORDER BY when OFFSET is present. Keeping the same
// fallback on the first page also prevents page boundaries from using
// different row orders when the table has no explicit key.
Some("1".to_string())
} else {
None
};
@@ -502,7 +507,7 @@ pub(super) fn build_select_columns(
.collect::<Vec<_>>()
.join(", ");
}
if database_type != Some(DatabaseType::Hive) {
if !matches!(database_type, Some(DatabaseType::Hive | DatabaseType::Impala)) {
return "*".to_string();
}
columns
+31
View File
@@ -4,11 +4,13 @@ use crate::models::connection::DatabaseType;
#[test]
fn transfer_identifier_policy_preserves_legacy_output() {
assert_eq!(quote_transfer_identifier("user`events", &DatabaseType::Hive), "`user``events`");
assert_eq!(quote_transfer_identifier("user`events", &DatabaseType::Impala), "`user``events`");
assert_eq!(quote_transfer_identifier("user`events", &DatabaseType::ClickHouse), "`user``events`");
assert_eq!(quote_transfer_identifier("user`events", &DatabaseType::Doris), "`user``events`");
assert_eq!(quote_transfer_identifier("user]events", &DatabaseType::SqlServer), "[user]]events]");
assert_eq!(quote_transfer_identifier("user\"events", &DatabaseType::Postgres), "\"user\"\"events\"");
assert_eq!(qualified_transfer_table("events", "warehouse", &DatabaseType::Hive, None), "`warehouse`.`events`");
assert_eq!(qualified_transfer_table("events", "warehouse", &DatabaseType::Impala, None), "`warehouse`.`events`");
assert_eq!(qualified_transfer_table("events", "warehouse", &DatabaseType::Mysql, None), "`events`");
}
@@ -245,6 +247,17 @@ fn builds_select_sql_with_limit_syntax_for_database_type() {
}),
"SELECT * FROM `test`.`dws_event_analyse` LIMIT 100;"
);
assert_eq!(
build_table_select_sql(TableSelectSqlOptions {
database_type: Some(DatabaseType::Impala),
schema: Some("dbx_demo"),
table_name: "connection_test",
columns: &[],
order_columns: &[],
limit: 100,
}),
"SELECT * FROM `dbx_demo`.`connection_test` LIMIT 100;"
);
assert_eq!(
build_table_select_sql(TableSelectSqlOptions {
database_type: Some(DatabaseType::StarRocks),
@@ -340,6 +353,24 @@ fn builds_table_data_where_and_schema_queries() {
}),
"SELECT * FROM \"public\".\"orders\" WHERE (amount > 10) LIMIT 50 OFFSET 100;"
);
assert_eq!(
build_table_data_select_sql(TableDataSelectSqlOptions {
database_type: Some(DatabaseType::Impala),
schema: Some("dbx_demo".to_string()),
table_name: "connection_test".to_string(),
table_type: Some("TABLE".to_string()),
primary_keys: Vec::new(),
columns: vec!["id".to_string(), "name".to_string()],
fallback_order_columns: Vec::new(),
order_by: None,
limit: Some(2),
offset: Some(1),
where_input: None,
include_row_id: false,
..Default::default()
}),
"SELECT `id` AS `id`, `name` AS `name` FROM `dbx_demo`.`connection_test` ORDER BY 1 LIMIT 2 OFFSET 1;"
);
assert_eq!(
build_table_data_select_sql(TableDataSelectSqlOptions {
database_type: Some(DatabaseType::Kingbase),
+281 -80
View File
@@ -5,10 +5,13 @@ use tokio::sync::RwLock;
use crate::connection::{config_for_pool_key, AppState, PoolKind};
use crate::db;
use crate::db::agent_driver::AgentTableReadStartParams;
use crate::db::mongo_driver::MongoDocumentResult;
use crate::models::connection::DatabaseType;
use crate::object_source_sql::{build_executable_object_source_statements, EditableObjectSourceSqlInput};
use crate::query::{agent_execute_query_params, pool_error_action, PoolErrorAction, QueryExecutionOptions};
use crate::query::{
agent_execute_query_params, pool_error_action, PoolErrorAction, QueryExecutionOptions, AGENT_PROTOCOL_MAX_ROWS,
};
use crate::sql::{split_sql_statements, split_sql_statements_for_database};
use crate::sql_dialect::{qualified_transfer_table, quote_transfer_identifier};
@@ -2652,7 +2655,7 @@ pub fn map_column_type(source_type: &str, _source_db: &DatabaseType, target_db:
// Extract basic type, `bigint unsigned` -> `bigint`
base = base.split(' ').next().unwrap_or(base).trim();
if matches!(target_db, DatabaseType::Hive) {
if matches!(target_db, DatabaseType::Hive | DatabaseType::Impala) {
return match base {
"tinyint" => "TINYINT".into(),
"smallint" | "int2" => "SMALLINT".into(),
@@ -2673,7 +2676,11 @@ pub fn map_column_type(source_type: &str, _source_db: &DatabaseType, target_db:
"TIMESTAMP".into()
}
"binary" | "varbinary" | "blob" | "tinyblob" | "mediumblob" | "longblob" | "bytea" | "image" => {
"BINARY".into()
if matches!(target_db, DatabaseType::Impala) {
"STRING".into()
} else {
"BINARY".into()
}
}
_ => "STRING".into(),
};
@@ -2852,7 +2859,7 @@ pub fn generate_create_table_ddl(
line.push(' ');
line.push_str(&default_clause);
}
if !c.is_nullable && !matches!(target_db, DatabaseType::Hive) {
if !c.is_nullable && !matches!(target_db, DatabaseType::Hive | DatabaseType::Impala) {
line.push_str(" NOT NULL");
}
if is_mysql_family {
@@ -2875,7 +2882,7 @@ pub fn generate_create_table_ddl(
}
let mut pks = Vec::with_capacity(columns.iter().filter(|c| c.is_primary_key).count());
if !matches!(target_db, DatabaseType::Hive) {
if !matches!(target_db, DatabaseType::Hive | DatabaseType::Impala) {
for c in columns {
if c.is_primary_key {
let qname = quote_identifier(&c.name, target_db);
@@ -3313,7 +3320,7 @@ fn generate_upsert_typed_for_transfer(
fn max_transfer_write_rows(db_type: &DatabaseType, mode: &TransferMode) -> usize {
match (db_type, mode) {
(DatabaseType::SqlServer, TransferMode::Append | TransferMode::Overwrite) => MAX_SQLSERVER_INSERT_ROWS,
(DatabaseType::Hive, _) => 500,
(DatabaseType::Hive | DatabaseType::Impala, _) => 500,
(DatabaseType::Oracle, TransferMode::Append | TransferMode::Overwrite) => MAX_ORACLE_INSERT_ALL_ROWS,
(DatabaseType::Oracle, TransferMode::Upsert) => MAX_ORACLE_MERGE_ROWS,
_ => usize::MAX,
@@ -6323,6 +6330,92 @@ where
Ok(total_transferred)
}
#[derive(Default)]
struct ImpalaTransferCursor {
started: bool,
session_id: Option<String>,
}
fn transfer_cursor_sql(
columns: &[String],
table: &str,
schema: &str,
db_type: &DatabaseType,
catalog: Option<&str>,
) -> String {
let full_table = qualified_table(table, schema, db_type, catalog);
let col_list = columns.iter().map(|column| quote_identifier(column, db_type)).collect::<Vec<_>>().join(", ");
format!("SELECT {col_list} FROM {full_table}")
}
async fn fetch_impala_transfer_batch(
state: &AppState,
pool_key: &str,
request: &TransferRequest,
sql: &str,
batch_size: usize,
cursor: &mut ImpalaTransferCursor,
) -> Result<db::QueryResult, String> {
let query_timeout_secs = if cursor.started {
0
} else {
let configs = state.configs.read().await;
configs.get(&request.source_connection_id).map(|config| config.query_timeout_secs).unwrap_or(0)
};
let connections = state.connections.read().await;
let Some(PoolKind::Agent(client)) = connections.get(pool_key) else {
return Err("Impala transfer requires an Agent connection".to_string());
};
let client = client.clone();
drop(connections);
let mut client = client.lock().await;
let result = if cursor.started {
let session_id =
cursor.session_id.as_deref().ok_or("Impala transfer cursor ended before the next page was requested")?;
client.fetch_table_read_page::<db::QueryResult>(session_id, batch_size).await?
} else {
cursor.started = true;
client
.start_table_read::<db::QueryResult>(AgentTableReadStartParams {
sql: sql.to_string(),
database: Some(request.source_database.clone()),
schema: (!request.source_schema.trim().is_empty()).then(|| request.source_schema.clone()),
page_size: batch_size,
max_rows: AGENT_PROTOCOL_MAX_ROWS,
fetch_size: Some(batch_size),
timeout_secs: (query_timeout_secs > 0).then_some(query_timeout_secs),
})
.await?
};
if result.has_more {
cursor.session_id = result.session_id.clone().or_else(|| cursor.session_id.clone());
if cursor.session_id.is_none() {
return Err("Impala transfer cursor did not return a session id for additional rows".to_string());
}
} else {
cursor.session_id = None;
}
Ok(result)
}
async fn close_impala_transfer_cursor(state: &AppState, pool_key: &str, cursor: &mut ImpalaTransferCursor) {
let Some(session_id) = cursor.session_id.take() else {
return;
};
let connections = state.connections.read().await;
let Some(PoolKind::Agent(client)) = connections.get(pool_key) else {
return;
};
let client = client.clone();
drop(connections);
let mut client = client.lock().await;
if let Err(error) = client.close_table_read_session::<bool>(&session_id).await {
log::warn!("[transfer] failed to close Impala transfer cursor: {error}");
}
}
/// Transfer a single table. Returns rows transferred.
/// `progress_callback` is invoked for progress updates.
#[allow(clippy::too_many_arguments)]
@@ -6683,7 +6776,7 @@ where
}
let target_columns = if (request.mode == TransferMode::Upsert
&& !matches!(target_db_type, DatabaseType::ClickHouse | DatabaseType::Hive))
&& !matches!(target_db_type, DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Impala))
|| matches!(target_db_type, DatabaseType::Postgres | DatabaseType::Dameng)
{
get_columns_for_transfer(
@@ -6703,7 +6796,7 @@ where
// Determine effective mode and PK columns for upsert
let (effective_mode, pk_columns) = if request.mode == TransferMode::Upsert {
if matches!(target_db_type, DatabaseType::ClickHouse | DatabaseType::Hive) {
if matches!(target_db_type, DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Impala) {
log::warn!("[transfer] upsert not supported for {:?}, falling back to append", target_db_type);
(TransferMode::Append, vec![])
} else {
@@ -6730,92 +6823,120 @@ where
let batch_size = if request.batch_size == 0 { 1000 } else { request.batch_size };
let mut offset: u64 = 0;
let mut total_transferred: u64 = 0;
loop {
if is_cancelled(&request.transfer_id).await {
return Err("Cancelled".to_string());
}
let sql = pagination_sql_with_order(
// A single Agent cursor keeps Impala rows in one query execution. Re-running
// LIMIT/OFFSET pages cannot be made stable for tables without a unique key.
let use_impala_cursor = matches!(source_db_type, DatabaseType::Impala);
let impala_transfer_sql = use_impala_cursor.then(|| {
transfer_cursor_sql(
&col_names,
table,
&request.source_schema,
source_db_type,
offset,
batch_size,
&primary_key_columns,
request.source_catalog.as_deref(),
);
let (sql, mysql_spatial_markers) =
mysql_spatial_transfer_select_sql(sql, &col_names, &col_types, source_db_type, target_db_type);
let result = execute_on_pool(state, source_pool_key, &sql).await?;
let row_count = result.rows.len();
)
});
let mut impala_cursor = ImpalaTransferCursor::default();
if row_count == 0 {
break;
}
let transfer_result: Result<(), String> = async {
loop {
if is_cancelled(&request.transfer_id).await {
return Err("Cancelled".to_string());
}
let write_statements = generate_transfer_write_sql_batches(
&effective_mode,
&col_names,
&col_types,
&result.rows,
&target_table,
&request.target_schema,
target_db_type,
&pk_columns,
request.target_catalog.as_deref(),
overrides_postgres_system_values,
mysql_spatial_markers,
)?;
for (statement_index, batch_sql) in write_statements.iter().enumerate() {
execute_transfer_write_statement(
state,
target_pool_key,
batch_sql,
target_db_type,
let (result, mysql_spatial_markers) = if let Some(sql) = impala_transfer_sql.as_deref() {
(
fetch_impala_transfer_batch(state, source_pool_key, request, sql, batch_size, &mut impala_cursor)
.await?,
false,
)
} else {
let sql = pagination_sql_with_order(
&col_names,
table,
&request.source_schema,
source_db_type,
offset,
batch_size,
&primary_key_columns,
request.source_catalog.as_deref(),
);
let (sql, mysql_spatial_markers) =
mysql_spatial_transfer_select_sql(sql, &col_names, &col_types, source_db_type, target_db_type);
(execute_on_pool(state, source_pool_key, &sql).await?, mysql_spatial_markers)
};
let has_more = result.has_more;
let row_count = result.rows.len();
if row_count == 0 {
break;
}
let write_statements = generate_transfer_write_sql_batches(
&effective_mode,
&col_names,
&col_types,
&result.rows,
&target_table,
&request.target_schema,
writes_dameng_identity_columns,
)
.await
.map_err(|e| {
let absolute_row = parse_mysql_row_error(&e).map(|row| offset + row);
match absolute_row {
Some(row) => format!(
"Insert failed for table '{target_table}' at row {row} (chunk {} of {}): {e}",
statement_index + 1,
write_statements.len()
),
None => format!(
"Insert failed for table '{target_table}' at offset {offset}, chunk {} of {}: {e}",
statement_index + 1,
write_statements.len()
),
}
})?;
}
target_db_type,
&pk_columns,
request.target_catalog.as_deref(),
overrides_postgres_system_values,
mysql_spatial_markers,
)?;
for (statement_index, batch_sql) in write_statements.iter().enumerate() {
execute_transfer_write_statement(
state,
target_pool_key,
batch_sql,
target_db_type,
&target_table,
&request.target_schema,
writes_dameng_identity_columns,
)
.await
.map_err(|e| {
let absolute_row = parse_mysql_row_error(&e).map(|row| offset + row);
match absolute_row {
Some(row) => format!(
"Insert failed for table '{target_table}' at row {row} (chunk {} of {}): {e}",
statement_index + 1,
write_statements.len()
),
None => format!(
"Insert failed for table '{target_table}' at offset {offset}, chunk {} of {}: {e}",
statement_index + 1,
write_statements.len()
),
}
})?;
}
total_transferred += row_count as u64;
log::info!("[transfer] {} batch +{} rows (total {})", table, row_count, total_transferred);
offset += row_count as u64;
total_transferred += row_count as u64;
log::info!("[transfer] {} batch +{} rows (total {})", table, row_count, total_transferred);
offset += row_count as u64;
progress_callback(TransferProgress {
transfer_id: request.transfer_id.clone(),
table: table.to_string(),
table_index,
total_tables,
rows_transferred: total_transferred,
total_rows,
status: TransferStatus::Running,
error: None,
terminal: false,
});
progress_callback(TransferProgress {
transfer_id: request.transfer_id.clone(),
table: table.to_string(),
table_index,
total_tables,
rows_transferred: total_transferred,
total_rows,
status: TransferStatus::Running,
error: None,
terminal: false,
});
if row_count < batch_size {
break;
if (use_impala_cursor && !has_more) || (!use_impala_cursor && row_count < batch_size) {
break;
}
}
Ok(())
}
.await;
close_impala_transfer_cursor(state, source_pool_key, &mut impala_cursor).await;
transfer_result?;
if pg_compat_transfer {
for statement in generate_postgres_sequence_sync_sql(&columns, &target_table, &request.target_schema) {
@@ -9214,6 +9335,54 @@ mod tests {
);
}
#[test]
fn impala_transfer_uses_impala_compatible_ddl_and_batches() {
let cols = vec![
db::ColumnInfo { is_primary_key: true, is_nullable: false, ..test_column("id", "bigint") },
db::ColumnInfo { is_nullable: false, ..test_column("payload", "jsonb") },
db::ColumnInfo { is_nullable: true, ..test_column("binary_payload", "bytea") },
];
let ddl = generate_create_table_ddl(
&cols,
"events",
"public",
"warehouse",
&DatabaseType::Impala,
&DatabaseType::Postgres,
None,
None,
);
assert!(ddl.contains("CREATE TABLE IF NOT EXISTS `warehouse`.`events`"));
assert!(ddl.contains("`id` BIGINT"));
assert!(ddl.contains("`payload` STRING"));
assert!(ddl.contains("`binary_payload` STRING"));
assert!(!ddl.contains("PRIMARY KEY"));
assert!(!ddl.contains("NOT NULL"));
assert_eq!(map_column_type("varbinary(255)", &DatabaseType::Mysql, &DatabaseType::Impala), "STRING");
assert_eq!(map_column_type("bytea", &DatabaseType::Postgres, &DatabaseType::Impala), "STRING");
assert_eq!(map_column_type("binary", &DatabaseType::Mysql, &DatabaseType::Hive), "BINARY");
let statements = generate_transfer_write_sql_batches(
&TransferMode::Append,
&[String::from("id"), String::from("binary_payload")],
&[Some(String::from("bigint")), Some(String::from("bytea"))],
&[vec![json!(1), json!("0x00ff")]],
"events",
"warehouse",
&DatabaseType::Impala,
&[],
None,
false,
false,
)
.unwrap();
assert_eq!(statements, vec!["INSERT INTO `warehouse`.`events` (`id`, `binary_payload`) VALUES\n(1, '0x00ff')"]);
assert_eq!(max_transfer_write_rows(&DatabaseType::Impala, &TransferMode::Append), 500);
assert_eq!(max_transfer_write_rows(&DatabaseType::Impala, &TransferMode::Upsert), 500);
}
#[test]
fn mongo_transfer_document_fields_preserve_first_seen_order() {
let documents = vec![json!({"b": 1}), json!({"a": 2, "c": 3}), json!({"b": 4, "d": 5})];
@@ -9260,6 +9429,38 @@ mod tests {
assert_eq!(sql, "SELECT \"id\", \"name\" FROM \"public\".\"users\" ORDER BY \"id\" LIMIT 100 OFFSET 200");
}
#[test]
fn impala_transfer_without_primary_key_uses_one_cursor_query() {
let sql = transfer_cursor_sql(
&[String::from("id"), String::from("name")],
"events",
"analytics",
&DatabaseType::Impala,
None,
);
assert_eq!(sql, "SELECT `id`, `name` FROM `analytics`.`events`");
assert!(!sql.contains("ORDER BY"));
assert!(!sql.contains("LIMIT"));
assert!(!sql.contains("OFFSET"));
}
#[test]
fn impala_transfer_preserves_explicit_primary_key_order() {
let sql = pagination_sql_with_order(
&[String::from("id"), String::from("name")],
"events",
"analytics",
&DatabaseType::Impala,
1000,
1000,
&[String::from("id")],
None,
);
assert_eq!(sql, "SELECT `id`, `name` FROM `analytics`.`events` ORDER BY `id` LIMIT 1000 OFFSET 1000");
}
#[test]
fn doris_unique_key_columns_drive_transfer_pagination_order() {
let columns =