feat(impala): add full database support
This commit is contained in:
54 files changed
+1092
-148
No files matched your search
@@ -196,6 +196,13 @@ const AGENT_CATALOG: &[AgentCatalogEntry] = &[
|
||||
store_visible: true,
|
||||
profiles: &[],
|
||||
},
|
||||
AgentCatalogEntry {
|
||||
db_type: DatabaseType::Impala,
|
||||
key: "hive",
|
||||
label: "Apache Impala",
|
||||
store_visible: false,
|
||||
profiles: &[],
|
||||
},
|
||||
AgentCatalogEntry {
|
||||
db_type: DatabaseType::Spark,
|
||||
key: "spark",
|
||||
@@ -397,4 +404,10 @@ mod tests {
|
||||
assert_eq!(label_for_key("duckdb"), Some("DuckDB"));
|
||||
assert!(!is_agent_type(&DatabaseType::DuckDb));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn impala_reuses_hive_agent_without_duplicate_store_entry() {
|
||||
assert_eq!(agent_key(&DatabaseType::Impala, None), Some("hive"));
|
||||
assert_eq!(driver_store_entries().filter(|(key, _)| *key == "hive").count(), 1);
|
||||
}
|
||||
}
|
||||
@@ -110,6 +110,7 @@ pub fn agent_connect_params_with_role(
|
||||
"jdbc_driver_class": agent_jdbc_driver_class(config),
|
||||
"jdbc_driver_paths": &config.jdbc_driver_paths,
|
||||
"sessionRole": session_role.as_str(),
|
||||
"database_type": config.db_type,
|
||||
});
|
||||
if config.db_type == DatabaseType::ZooKeeper {
|
||||
params["connection_timeout_ms"] = serde_json::json!(
|
||||
@@ -700,6 +701,7 @@ mod tests {
|
||||
fn agent_connect_params_default_to_workload_session_role() {
|
||||
let params = agent_connect_params(&config(DatabaseType::H2, Some("test")), "127.0.0.1", 9092, "test");
|
||||
assert_eq!(params["sessionRole"], "workload");
|
||||
assert_eq!(params["database_type"], "h2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -172,6 +172,7 @@ macro_rules! agent_connection_pool_database_type {
|
||||
| DatabaseType::Snowflake
|
||||
| DatabaseType::Trino
|
||||
| DatabaseType::Hive
|
||||
| DatabaseType::Impala
|
||||
| DatabaseType::Spark
|
||||
| DatabaseType::Db2
|
||||
| DatabaseType::Informix
|
||||
|
||||
@@ -645,7 +645,7 @@ pub fn build_duplicate_table_structure_sql(options: DuplicateTableStructureSqlOp
|
||||
let source = qualified_name(options.database_type, options.schema.as_deref(), &options.source_name);
|
||||
let target =
|
||||
qualified_duplicate_target_name(options.database_type, options.schema.as_deref(), &options.target_name);
|
||||
let structure_sql = if options.database_type == Some(DatabaseType::Mysql) {
|
||||
let structure_sql = if matches!(options.database_type, Some(DatabaseType::Mysql | DatabaseType::Impala)) {
|
||||
format!("CREATE TABLE {target} LIKE {source};")
|
||||
} else if options.database_type == Some(DatabaseType::Questdb) {
|
||||
format!("CREATE TABLE {target} (LIKE {source});")
|
||||
@@ -1684,6 +1684,17 @@ mod tests {
|
||||
}),
|
||||
"CREATE TABLE \"public\".\"users_copy\" (LIKE \"public\".\"users\" INCLUDING ALL);"
|
||||
);
|
||||
assert_eq!(
|
||||
build_duplicate_table_structure_sql(DuplicateTableStructureSqlOptions {
|
||||
database_type: Some(DatabaseType::Impala),
|
||||
schema: Some("dbx_demo".to_string()),
|
||||
source_name: "connection_test".to_string(),
|
||||
target_name: "connection_test_copy".to_string(),
|
||||
table_comment: None,
|
||||
column_comments: vec![],
|
||||
}),
|
||||
"CREATE TABLE `dbx_demo`.`connection_test_copy` LIKE `dbx_demo`.`connection_test`;"
|
||||
);
|
||||
assert_eq!(
|
||||
build_duplicate_table_structure_sql(DuplicateTableStructureSqlOptions {
|
||||
database_type: Some(DatabaseType::Postgres),
|
||||
|
||||
@@ -564,6 +564,7 @@ pub enum DatabaseType {
|
||||
#[serde(rename = "prestosql")]
|
||||
PrestoSql,
|
||||
Hive,
|
||||
Impala,
|
||||
Spark,
|
||||
#[serde(rename = "db2")]
|
||||
Db2,
|
||||
@@ -1141,6 +1142,7 @@ impl ConnectionConfig {
|
||||
DatabaseType::Trino => format!("trino://{host}:{port}{db_part}"),
|
||||
DatabaseType::PrestoSql => format!("prestosql://{host}:{port}{db_part}"),
|
||||
DatabaseType::Hive => format!("hive://{host}:{port}{db_part}"),
|
||||
DatabaseType::Impala => format!("impala://{host}:{port}{db_part}"),
|
||||
DatabaseType::Spark => format!("spark://{host}:{port}{db_part}"),
|
||||
DatabaseType::Db2 => format!("db2://{host}:{port}{db_part}"),
|
||||
DatabaseType::Informix => format!("informix://{host}:{port}{db_part}"),
|
||||
@@ -1351,6 +1353,13 @@ impl ConnectionConfig {
|
||||
DatabaseType::Hive => {
|
||||
format!("hive://{}:{}@{host}:{port}{db_part}", username, password)
|
||||
}
|
||||
DatabaseType::Impala => {
|
||||
if self.username.is_empty() && self.password.is_empty() {
|
||||
format!("impala://{host}:{port}{db_part}")
|
||||
} else {
|
||||
format!("impala://{}:{}@{host}:{port}{db_part}", username, password)
|
||||
}
|
||||
}
|
||||
DatabaseType::Spark => {
|
||||
format!("spark://{}:{}@{host}:{port}{db_part}", username, password)
|
||||
}
|
||||
@@ -2599,6 +2608,25 @@ mod tests {
|
||||
assert_eq!(serde_json::from_str::<DatabaseType>("\"easysearch\"").unwrap(), DatabaseType::Easysearch);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn impala_database_type_and_connection_urls_are_stable() {
|
||||
assert_eq!(serde_json::to_string(&DatabaseType::Impala).unwrap(), "\"impala\"");
|
||||
assert_eq!(serde_json::from_str::<DatabaseType>("\"impala\"").unwrap(), DatabaseType::Impala);
|
||||
|
||||
let mut config = mysql_config("", "", Some("analytics"));
|
||||
config.db_type = DatabaseType::Impala;
|
||||
config.host = "impala.local".to_string();
|
||||
config.port = 21050;
|
||||
assert_eq!(config.connection_url(), "impala://impala.local:21050/analytics");
|
||||
|
||||
config.username = "analyst".to_string();
|
||||
config.password = "secret".to_string();
|
||||
assert_eq!(
|
||||
config.connection_url_with_host(&config.host, config.port),
|
||||
"impala://analyst:secret@impala.local:21050/analytics"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cloudflare_d1_database_type_and_api_url_are_stable() {
|
||||
assert_eq!(serde_json::to_string(&DatabaseType::CloudflareD1).unwrap(), "\"cloudflare-d1\"");
|
||||
|
||||
@@ -53,6 +53,7 @@ pub fn is_schema_aware(database_type: DatabaseType) -> bool {
|
||||
| DatabaseType::Trino
|
||||
| DatabaseType::PrestoSql
|
||||
| DatabaseType::Hive
|
||||
| DatabaseType::Impala
|
||||
| DatabaseType::Spark
|
||||
| DatabaseType::Db2
|
||||
| DatabaseType::Informix
|
||||
|
||||
@@ -669,8 +669,8 @@ pub fn profile_for(db_type: DatabaseType) -> DdlDialectProfile {
|
||||
|
||||
ClickHouse => clickhouse_profile(),
|
||||
|
||||
DuckDb | Questdb | SapHana | Teradata | Snowflake | Trino | PrestoSql | Hive | Spark | Db2 | Informix
|
||||
| Bigquery | Kylin | Oscar | Tdengine | Iotdb | Databricks | Jdbc => conservative_ansi(db_type),
|
||||
DuckDb | Questdb | SapHana | Teradata | Snowflake | Trino | PrestoSql | Hive | Impala | Spark | Db2
|
||||
| Informix | Bigquery | Kylin | Oscar | Tdengine | Iotdb | Databricks | Jdbc => conservative_ansi(db_type),
|
||||
|
||||
// Non-tabular / not applicable for relational CREATE TABLE
|
||||
Redis | MongoDb | Elasticsearch | Easysearch | Meilisearch | Qdrant | Milvus | Weaviate | ChromaDb | Neo4j
|
||||
|
||||
@@ -129,6 +129,7 @@ pub fn quote_table_identifier(database_type: Option<DatabaseType>, name: &str) -
|
||||
| DatabaseType::StarRocks
|
||||
| DatabaseType::ManticoreSearch
|
||||
| DatabaseType::Hive
|
||||
| DatabaseType::Impala
|
||||
| DatabaseType::Spark
|
||||
| DatabaseType::Databend
|
||||
| DatabaseType::Tdengine
|
||||
@@ -378,6 +379,7 @@ pub(crate) fn quote_transfer_identifier(name: &str, database_type: &DatabaseType
|
||||
| DatabaseType::Doris
|
||||
| DatabaseType::StarRocks
|
||||
| DatabaseType::Hive
|
||||
| DatabaseType::Impala
|
||||
| DatabaseType::Spark
|
||||
| DatabaseType::Questdb => format!("`{}`", name.replace('`', "``")),
|
||||
DatabaseType::SqlServer => format!("[{}]", name.replace(']', "]]")),
|
||||
|
||||
@@ -186,6 +186,11 @@ pub fn build_table_data_select_sql(options: TableDataSelectSqlOptions) -> String
|
||||
let default_order_by = if database_type == Some(DatabaseType::InfluxDb) {
|
||||
// InfluxQL only allows sorting of timestamp column
|
||||
Some("time DESC".to_string())
|
||||
} else if database_type == Some(DatabaseType::Impala) {
|
||||
// Impala requires ORDER BY when OFFSET is present. Keeping the same
|
||||
// fallback on the first page also prevents page boundaries from using
|
||||
// different row orders when the table has no explicit key.
|
||||
Some("1".to_string())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -502,7 +507,7 @@ pub(super) fn build_select_columns(
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
}
|
||||
if database_type != Some(DatabaseType::Hive) {
|
||||
if !matches!(database_type, Some(DatabaseType::Hive | DatabaseType::Impala)) {
|
||||
return "*".to_string();
|
||||
}
|
||||
columns
|
||||
|
||||
@@ -4,11 +4,13 @@ use crate::models::connection::DatabaseType;
|
||||
#[test]
|
||||
fn transfer_identifier_policy_preserves_legacy_output() {
|
||||
assert_eq!(quote_transfer_identifier("user`events", &DatabaseType::Hive), "`user``events`");
|
||||
assert_eq!(quote_transfer_identifier("user`events", &DatabaseType::Impala), "`user``events`");
|
||||
assert_eq!(quote_transfer_identifier("user`events", &DatabaseType::ClickHouse), "`user``events`");
|
||||
assert_eq!(quote_transfer_identifier("user`events", &DatabaseType::Doris), "`user``events`");
|
||||
assert_eq!(quote_transfer_identifier("user]events", &DatabaseType::SqlServer), "[user]]events]");
|
||||
assert_eq!(quote_transfer_identifier("user\"events", &DatabaseType::Postgres), "\"user\"\"events\"");
|
||||
assert_eq!(qualified_transfer_table("events", "warehouse", &DatabaseType::Hive, None), "`warehouse`.`events`");
|
||||
assert_eq!(qualified_transfer_table("events", "warehouse", &DatabaseType::Impala, None), "`warehouse`.`events`");
|
||||
assert_eq!(qualified_transfer_table("events", "warehouse", &DatabaseType::Mysql, None), "`events`");
|
||||
}
|
||||
|
||||
@@ -245,6 +247,17 @@ fn builds_select_sql_with_limit_syntax_for_database_type() {
|
||||
}),
|
||||
"SELECT * FROM `test`.`dws_event_analyse` LIMIT 100;"
|
||||
);
|
||||
assert_eq!(
|
||||
build_table_select_sql(TableSelectSqlOptions {
|
||||
database_type: Some(DatabaseType::Impala),
|
||||
schema: Some("dbx_demo"),
|
||||
table_name: "connection_test",
|
||||
columns: &[],
|
||||
order_columns: &[],
|
||||
limit: 100,
|
||||
}),
|
||||
"SELECT * FROM `dbx_demo`.`connection_test` LIMIT 100;"
|
||||
);
|
||||
assert_eq!(
|
||||
build_table_select_sql(TableSelectSqlOptions {
|
||||
database_type: Some(DatabaseType::StarRocks),
|
||||
@@ -340,6 +353,24 @@ fn builds_table_data_where_and_schema_queries() {
|
||||
}),
|
||||
"SELECT * FROM \"public\".\"orders\" WHERE (amount > 10) LIMIT 50 OFFSET 100;"
|
||||
);
|
||||
assert_eq!(
|
||||
build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||
database_type: Some(DatabaseType::Impala),
|
||||
schema: Some("dbx_demo".to_string()),
|
||||
table_name: "connection_test".to_string(),
|
||||
table_type: Some("TABLE".to_string()),
|
||||
primary_keys: Vec::new(),
|
||||
columns: vec!["id".to_string(), "name".to_string()],
|
||||
fallback_order_columns: Vec::new(),
|
||||
order_by: None,
|
||||
limit: Some(2),
|
||||
offset: Some(1),
|
||||
where_input: None,
|
||||
include_row_id: false,
|
||||
..Default::default()
|
||||
}),
|
||||
"SELECT `id` AS `id`, `name` AS `name` FROM `dbx_demo`.`connection_test` ORDER BY 1 LIMIT 2 OFFSET 1;"
|
||||
);
|
||||
assert_eq!(
|
||||
build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||
database_type: Some(DatabaseType::Kingbase),
|
||||
|
||||
+281
-80
@@ -5,10 +5,13 @@ use tokio::sync::RwLock;
|
||||
|
||||
use crate::connection::{config_for_pool_key, AppState, PoolKind};
|
||||
use crate::db;
|
||||
use crate::db::agent_driver::AgentTableReadStartParams;
|
||||
use crate::db::mongo_driver::MongoDocumentResult;
|
||||
use crate::models::connection::DatabaseType;
|
||||
use crate::object_source_sql::{build_executable_object_source_statements, EditableObjectSourceSqlInput};
|
||||
use crate::query::{agent_execute_query_params, pool_error_action, PoolErrorAction, QueryExecutionOptions};
|
||||
use crate::query::{
|
||||
agent_execute_query_params, pool_error_action, PoolErrorAction, QueryExecutionOptions, AGENT_PROTOCOL_MAX_ROWS,
|
||||
};
|
||||
use crate::sql::{split_sql_statements, split_sql_statements_for_database};
|
||||
use crate::sql_dialect::{qualified_transfer_table, quote_transfer_identifier};
|
||||
|
||||
@@ -2652,7 +2655,7 @@ pub fn map_column_type(source_type: &str, _source_db: &DatabaseType, target_db:
|
||||
// Extract basic type, `bigint unsigned` -> `bigint`
|
||||
base = base.split(' ').next().unwrap_or(base).trim();
|
||||
|
||||
if matches!(target_db, DatabaseType::Hive) {
|
||||
if matches!(target_db, DatabaseType::Hive | DatabaseType::Impala) {
|
||||
return match base {
|
||||
"tinyint" => "TINYINT".into(),
|
||||
"smallint" | "int2" => "SMALLINT".into(),
|
||||
@@ -2673,7 +2676,11 @@ pub fn map_column_type(source_type: &str, _source_db: &DatabaseType, target_db:
|
||||
"TIMESTAMP".into()
|
||||
}
|
||||
"binary" | "varbinary" | "blob" | "tinyblob" | "mediumblob" | "longblob" | "bytea" | "image" => {
|
||||
"BINARY".into()
|
||||
if matches!(target_db, DatabaseType::Impala) {
|
||||
"STRING".into()
|
||||
} else {
|
||||
"BINARY".into()
|
||||
}
|
||||
}
|
||||
_ => "STRING".into(),
|
||||
};
|
||||
@@ -2852,7 +2859,7 @@ pub fn generate_create_table_ddl(
|
||||
line.push(' ');
|
||||
line.push_str(&default_clause);
|
||||
}
|
||||
if !c.is_nullable && !matches!(target_db, DatabaseType::Hive) {
|
||||
if !c.is_nullable && !matches!(target_db, DatabaseType::Hive | DatabaseType::Impala) {
|
||||
line.push_str(" NOT NULL");
|
||||
}
|
||||
if is_mysql_family {
|
||||
@@ -2875,7 +2882,7 @@ pub fn generate_create_table_ddl(
|
||||
}
|
||||
|
||||
let mut pks = Vec::with_capacity(columns.iter().filter(|c| c.is_primary_key).count());
|
||||
if !matches!(target_db, DatabaseType::Hive) {
|
||||
if !matches!(target_db, DatabaseType::Hive | DatabaseType::Impala) {
|
||||
for c in columns {
|
||||
if c.is_primary_key {
|
||||
let qname = quote_identifier(&c.name, target_db);
|
||||
@@ -3313,7 +3320,7 @@ fn generate_upsert_typed_for_transfer(
|
||||
fn max_transfer_write_rows(db_type: &DatabaseType, mode: &TransferMode) -> usize {
|
||||
match (db_type, mode) {
|
||||
(DatabaseType::SqlServer, TransferMode::Append | TransferMode::Overwrite) => MAX_SQLSERVER_INSERT_ROWS,
|
||||
(DatabaseType::Hive, _) => 500,
|
||||
(DatabaseType::Hive | DatabaseType::Impala, _) => 500,
|
||||
(DatabaseType::Oracle, TransferMode::Append | TransferMode::Overwrite) => MAX_ORACLE_INSERT_ALL_ROWS,
|
||||
(DatabaseType::Oracle, TransferMode::Upsert) => MAX_ORACLE_MERGE_ROWS,
|
||||
_ => usize::MAX,
|
||||
@@ -6323,6 +6330,92 @@ where
|
||||
Ok(total_transferred)
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ImpalaTransferCursor {
|
||||
started: bool,
|
||||
session_id: Option<String>,
|
||||
}
|
||||
|
||||
fn transfer_cursor_sql(
|
||||
columns: &[String],
|
||||
table: &str,
|
||||
schema: &str,
|
||||
db_type: &DatabaseType,
|
||||
catalog: Option<&str>,
|
||||
) -> String {
|
||||
let full_table = qualified_table(table, schema, db_type, catalog);
|
||||
let col_list = columns.iter().map(|column| quote_identifier(column, db_type)).collect::<Vec<_>>().join(", ");
|
||||
format!("SELECT {col_list} FROM {full_table}")
|
||||
}
|
||||
|
||||
async fn fetch_impala_transfer_batch(
|
||||
state: &AppState,
|
||||
pool_key: &str,
|
||||
request: &TransferRequest,
|
||||
sql: &str,
|
||||
batch_size: usize,
|
||||
cursor: &mut ImpalaTransferCursor,
|
||||
) -> Result<db::QueryResult, String> {
|
||||
let query_timeout_secs = if cursor.started {
|
||||
0
|
||||
} else {
|
||||
let configs = state.configs.read().await;
|
||||
configs.get(&request.source_connection_id).map(|config| config.query_timeout_secs).unwrap_or(0)
|
||||
};
|
||||
let connections = state.connections.read().await;
|
||||
let Some(PoolKind::Agent(client)) = connections.get(pool_key) else {
|
||||
return Err("Impala transfer requires an Agent connection".to_string());
|
||||
};
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
|
||||
let mut client = client.lock().await;
|
||||
let result = if cursor.started {
|
||||
let session_id =
|
||||
cursor.session_id.as_deref().ok_or("Impala transfer cursor ended before the next page was requested")?;
|
||||
client.fetch_table_read_page::<db::QueryResult>(session_id, batch_size).await?
|
||||
} else {
|
||||
cursor.started = true;
|
||||
client
|
||||
.start_table_read::<db::QueryResult>(AgentTableReadStartParams {
|
||||
sql: sql.to_string(),
|
||||
database: Some(request.source_database.clone()),
|
||||
schema: (!request.source_schema.trim().is_empty()).then(|| request.source_schema.clone()),
|
||||
page_size: batch_size,
|
||||
max_rows: AGENT_PROTOCOL_MAX_ROWS,
|
||||
fetch_size: Some(batch_size),
|
||||
timeout_secs: (query_timeout_secs > 0).then_some(query_timeout_secs),
|
||||
})
|
||||
.await?
|
||||
};
|
||||
|
||||
if result.has_more {
|
||||
cursor.session_id = result.session_id.clone().or_else(|| cursor.session_id.clone());
|
||||
if cursor.session_id.is_none() {
|
||||
return Err("Impala transfer cursor did not return a session id for additional rows".to_string());
|
||||
}
|
||||
} else {
|
||||
cursor.session_id = None;
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn close_impala_transfer_cursor(state: &AppState, pool_key: &str, cursor: &mut ImpalaTransferCursor) {
|
||||
let Some(session_id) = cursor.session_id.take() else {
|
||||
return;
|
||||
};
|
||||
let connections = state.connections.read().await;
|
||||
let Some(PoolKind::Agent(client)) = connections.get(pool_key) else {
|
||||
return;
|
||||
};
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
if let Err(error) = client.close_table_read_session::<bool>(&session_id).await {
|
||||
log::warn!("[transfer] failed to close Impala transfer cursor: {error}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Transfer a single table. Returns rows transferred.
|
||||
/// `progress_callback` is invoked for progress updates.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -6683,7 +6776,7 @@ where
|
||||
}
|
||||
|
||||
let target_columns = if (request.mode == TransferMode::Upsert
|
||||
&& !matches!(target_db_type, DatabaseType::ClickHouse | DatabaseType::Hive))
|
||||
&& !matches!(target_db_type, DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Impala))
|
||||
|| matches!(target_db_type, DatabaseType::Postgres | DatabaseType::Dameng)
|
||||
{
|
||||
get_columns_for_transfer(
|
||||
@@ -6703,7 +6796,7 @@ where
|
||||
|
||||
// Determine effective mode and PK columns for upsert
|
||||
let (effective_mode, pk_columns) = if request.mode == TransferMode::Upsert {
|
||||
if matches!(target_db_type, DatabaseType::ClickHouse | DatabaseType::Hive) {
|
||||
if matches!(target_db_type, DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Impala) {
|
||||
log::warn!("[transfer] upsert not supported for {:?}, falling back to append", target_db_type);
|
||||
(TransferMode::Append, vec![])
|
||||
} else {
|
||||
@@ -6730,92 +6823,120 @@ where
|
||||
let batch_size = if request.batch_size == 0 { 1000 } else { request.batch_size };
|
||||
let mut offset: u64 = 0;
|
||||
let mut total_transferred: u64 = 0;
|
||||
|
||||
loop {
|
||||
if is_cancelled(&request.transfer_id).await {
|
||||
return Err("Cancelled".to_string());
|
||||
}
|
||||
|
||||
let sql = pagination_sql_with_order(
|
||||
// A single Agent cursor keeps Impala rows in one query execution. Re-running
|
||||
// LIMIT/OFFSET pages cannot be made stable for tables without a unique key.
|
||||
let use_impala_cursor = matches!(source_db_type, DatabaseType::Impala);
|
||||
let impala_transfer_sql = use_impala_cursor.then(|| {
|
||||
transfer_cursor_sql(
|
||||
&col_names,
|
||||
table,
|
||||
&request.source_schema,
|
||||
source_db_type,
|
||||
offset,
|
||||
batch_size,
|
||||
&primary_key_columns,
|
||||
request.source_catalog.as_deref(),
|
||||
);
|
||||
let (sql, mysql_spatial_markers) =
|
||||
mysql_spatial_transfer_select_sql(sql, &col_names, &col_types, source_db_type, target_db_type);
|
||||
let result = execute_on_pool(state, source_pool_key, &sql).await?;
|
||||
let row_count = result.rows.len();
|
||||
)
|
||||
});
|
||||
let mut impala_cursor = ImpalaTransferCursor::default();
|
||||
|
||||
if row_count == 0 {
|
||||
break;
|
||||
}
|
||||
let transfer_result: Result<(), String> = async {
|
||||
loop {
|
||||
if is_cancelled(&request.transfer_id).await {
|
||||
return Err("Cancelled".to_string());
|
||||
}
|
||||
|
||||
let write_statements = generate_transfer_write_sql_batches(
|
||||
&effective_mode,
|
||||
&col_names,
|
||||
&col_types,
|
||||
&result.rows,
|
||||
&target_table,
|
||||
&request.target_schema,
|
||||
target_db_type,
|
||||
&pk_columns,
|
||||
request.target_catalog.as_deref(),
|
||||
overrides_postgres_system_values,
|
||||
mysql_spatial_markers,
|
||||
)?;
|
||||
for (statement_index, batch_sql) in write_statements.iter().enumerate() {
|
||||
execute_transfer_write_statement(
|
||||
state,
|
||||
target_pool_key,
|
||||
batch_sql,
|
||||
target_db_type,
|
||||
let (result, mysql_spatial_markers) = if let Some(sql) = impala_transfer_sql.as_deref() {
|
||||
(
|
||||
fetch_impala_transfer_batch(state, source_pool_key, request, sql, batch_size, &mut impala_cursor)
|
||||
.await?,
|
||||
false,
|
||||
)
|
||||
} else {
|
||||
let sql = pagination_sql_with_order(
|
||||
&col_names,
|
||||
table,
|
||||
&request.source_schema,
|
||||
source_db_type,
|
||||
offset,
|
||||
batch_size,
|
||||
&primary_key_columns,
|
||||
request.source_catalog.as_deref(),
|
||||
);
|
||||
let (sql, mysql_spatial_markers) =
|
||||
mysql_spatial_transfer_select_sql(sql, &col_names, &col_types, source_db_type, target_db_type);
|
||||
(execute_on_pool(state, source_pool_key, &sql).await?, mysql_spatial_markers)
|
||||
};
|
||||
let has_more = result.has_more;
|
||||
let row_count = result.rows.len();
|
||||
|
||||
if row_count == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
let write_statements = generate_transfer_write_sql_batches(
|
||||
&effective_mode,
|
||||
&col_names,
|
||||
&col_types,
|
||||
&result.rows,
|
||||
&target_table,
|
||||
&request.target_schema,
|
||||
writes_dameng_identity_columns,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
let absolute_row = parse_mysql_row_error(&e).map(|row| offset + row);
|
||||
match absolute_row {
|
||||
Some(row) => format!(
|
||||
"Insert failed for table '{target_table}' at row {row} (chunk {} of {}): {e}",
|
||||
statement_index + 1,
|
||||
write_statements.len()
|
||||
),
|
||||
None => format!(
|
||||
"Insert failed for table '{target_table}' at offset {offset}, chunk {} of {}: {e}",
|
||||
statement_index + 1,
|
||||
write_statements.len()
|
||||
),
|
||||
}
|
||||
})?;
|
||||
}
|
||||
target_db_type,
|
||||
&pk_columns,
|
||||
request.target_catalog.as_deref(),
|
||||
overrides_postgres_system_values,
|
||||
mysql_spatial_markers,
|
||||
)?;
|
||||
for (statement_index, batch_sql) in write_statements.iter().enumerate() {
|
||||
execute_transfer_write_statement(
|
||||
state,
|
||||
target_pool_key,
|
||||
batch_sql,
|
||||
target_db_type,
|
||||
&target_table,
|
||||
&request.target_schema,
|
||||
writes_dameng_identity_columns,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
let absolute_row = parse_mysql_row_error(&e).map(|row| offset + row);
|
||||
match absolute_row {
|
||||
Some(row) => format!(
|
||||
"Insert failed for table '{target_table}' at row {row} (chunk {} of {}): {e}",
|
||||
statement_index + 1,
|
||||
write_statements.len()
|
||||
),
|
||||
None => format!(
|
||||
"Insert failed for table '{target_table}' at offset {offset}, chunk {} of {}: {e}",
|
||||
statement_index + 1,
|
||||
write_statements.len()
|
||||
),
|
||||
}
|
||||
})?;
|
||||
}
|
||||
|
||||
total_transferred += row_count as u64;
|
||||
log::info!("[transfer] {} batch +{} rows (total {})", table, row_count, total_transferred);
|
||||
offset += row_count as u64;
|
||||
total_transferred += row_count as u64;
|
||||
log::info!("[transfer] {} batch +{} rows (total {})", table, row_count, total_transferred);
|
||||
offset += row_count as u64;
|
||||
|
||||
progress_callback(TransferProgress {
|
||||
transfer_id: request.transfer_id.clone(),
|
||||
table: table.to_string(),
|
||||
table_index,
|
||||
total_tables,
|
||||
rows_transferred: total_transferred,
|
||||
total_rows,
|
||||
status: TransferStatus::Running,
|
||||
error: None,
|
||||
terminal: false,
|
||||
});
|
||||
progress_callback(TransferProgress {
|
||||
transfer_id: request.transfer_id.clone(),
|
||||
table: table.to_string(),
|
||||
table_index,
|
||||
total_tables,
|
||||
rows_transferred: total_transferred,
|
||||
total_rows,
|
||||
status: TransferStatus::Running,
|
||||
error: None,
|
||||
terminal: false,
|
||||
});
|
||||
|
||||
if row_count < batch_size {
|
||||
break;
|
||||
if (use_impala_cursor && !has_more) || (!use_impala_cursor && row_count < batch_size) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
close_impala_transfer_cursor(state, source_pool_key, &mut impala_cursor).await;
|
||||
transfer_result?;
|
||||
|
||||
if pg_compat_transfer {
|
||||
for statement in generate_postgres_sequence_sync_sql(&columns, &target_table, &request.target_schema) {
|
||||
@@ -9214,6 +9335,54 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn impala_transfer_uses_impala_compatible_ddl_and_batches() {
|
||||
let cols = vec![
|
||||
db::ColumnInfo { is_primary_key: true, is_nullable: false, ..test_column("id", "bigint") },
|
||||
db::ColumnInfo { is_nullable: false, ..test_column("payload", "jsonb") },
|
||||
db::ColumnInfo { is_nullable: true, ..test_column("binary_payload", "bytea") },
|
||||
];
|
||||
|
||||
let ddl = generate_create_table_ddl(
|
||||
&cols,
|
||||
"events",
|
||||
"public",
|
||||
"warehouse",
|
||||
&DatabaseType::Impala,
|
||||
&DatabaseType::Postgres,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
assert!(ddl.contains("CREATE TABLE IF NOT EXISTS `warehouse`.`events`"));
|
||||
assert!(ddl.contains("`id` BIGINT"));
|
||||
assert!(ddl.contains("`payload` STRING"));
|
||||
assert!(ddl.contains("`binary_payload` STRING"));
|
||||
assert!(!ddl.contains("PRIMARY KEY"));
|
||||
assert!(!ddl.contains("NOT NULL"));
|
||||
assert_eq!(map_column_type("varbinary(255)", &DatabaseType::Mysql, &DatabaseType::Impala), "STRING");
|
||||
assert_eq!(map_column_type("bytea", &DatabaseType::Postgres, &DatabaseType::Impala), "STRING");
|
||||
assert_eq!(map_column_type("binary", &DatabaseType::Mysql, &DatabaseType::Hive), "BINARY");
|
||||
|
||||
let statements = generate_transfer_write_sql_batches(
|
||||
&TransferMode::Append,
|
||||
&[String::from("id"), String::from("binary_payload")],
|
||||
&[Some(String::from("bigint")), Some(String::from("bytea"))],
|
||||
&[vec![json!(1), json!("0x00ff")]],
|
||||
"events",
|
||||
"warehouse",
|
||||
&DatabaseType::Impala,
|
||||
&[],
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(statements, vec!["INSERT INTO `warehouse`.`events` (`id`, `binary_payload`) VALUES\n(1, '0x00ff')"]);
|
||||
assert_eq!(max_transfer_write_rows(&DatabaseType::Impala, &TransferMode::Append), 500);
|
||||
assert_eq!(max_transfer_write_rows(&DatabaseType::Impala, &TransferMode::Upsert), 500);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mongo_transfer_document_fields_preserve_first_seen_order() {
|
||||
let documents = vec![json!({"b": 1}), json!({"a": 2, "c": 3}), json!({"b": 4, "d": 5})];
|
||||
@@ -9260,6 +9429,38 @@ mod tests {
|
||||
assert_eq!(sql, "SELECT \"id\", \"name\" FROM \"public\".\"users\" ORDER BY \"id\" LIMIT 100 OFFSET 200");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn impala_transfer_without_primary_key_uses_one_cursor_query() {
|
||||
let sql = transfer_cursor_sql(
|
||||
&[String::from("id"), String::from("name")],
|
||||
"events",
|
||||
"analytics",
|
||||
&DatabaseType::Impala,
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(sql, "SELECT `id`, `name` FROM `analytics`.`events`");
|
||||
assert!(!sql.contains("ORDER BY"));
|
||||
assert!(!sql.contains("LIMIT"));
|
||||
assert!(!sql.contains("OFFSET"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn impala_transfer_preserves_explicit_primary_key_order() {
|
||||
let sql = pagination_sql_with_order(
|
||||
&[String::from("id"), String::from("name")],
|
||||
"events",
|
||||
"analytics",
|
||||
&DatabaseType::Impala,
|
||||
1000,
|
||||
1000,
|
||||
&[String::from("id")],
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(sql, "SELECT `id`, `name` FROM `analytics`.`events` ORDER BY `id` LIMIT 1000 OFFSET 1000");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn doris_unique_key_columns_drive_transfer_pagination_order() {
|
||||
let columns =
|
||||
|
||||
Reference in new issue
Block a user