fix(mysql): stream query result exports
This commit is contained in:
1 parent
9cbe857a2a
commit
3609641fc8
3 files changed
+412
-9
No files matched your search
@@ -1,9 +1,14 @@
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use dbx_core::connection::AppState;
|
||||
use dbx_core::models::connection::{ConnectionConfig, DatabaseType};
|
||||
use dbx_core::query::execute_sql_statement;
|
||||
use dbx_core::query_result_export::{export_query_result_core, ExportStatus, QueryResultExportRequest};
|
||||
use dbx_core::sql::{split_sql_statements_for_database, SqlFileRequest};
|
||||
use dbx_core::sql_file_import::execute_sql_file_content;
|
||||
use dbx_core::storage::Storage;
|
||||
use dbx_core::table_import::parse_xlsx_file;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
fn live_mysql_sql_file_config(id: &str) -> ConnectionConfig {
|
||||
@@ -38,6 +43,41 @@ async fn app_state_with_config(config: ConnectionConfig) -> (AppState, std::path
|
||||
(state, db_path)
|
||||
}
|
||||
|
||||
fn live_mysql_query_export_config(
|
||||
id: &str,
|
||||
host: &str,
|
||||
port: u16,
|
||||
user: &str,
|
||||
password: &str,
|
||||
database: &str,
|
||||
) -> ConnectionConfig {
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"id": id,
|
||||
"name": id,
|
||||
"db_type": DatabaseType::Mysql,
|
||||
"host": host,
|
||||
"port": port,
|
||||
"username": user,
|
||||
"password": password,
|
||||
"database": database,
|
||||
"connect_timeout_secs": 10,
|
||||
"query_timeout_secs": 30,
|
||||
"idle_timeout_secs": 60,
|
||||
"keepalive_interval_secs": 0
|
||||
}))
|
||||
.expect("live MySQL query export config should deserialize")
|
||||
}
|
||||
|
||||
fn json_cell_text(value: &serde_json::Value) -> String {
|
||||
match value {
|
||||
serde_json::Value::String(value) => value.clone(),
|
||||
serde_json::Value::Number(value) => value.to_string(),
|
||||
serde_json::Value::Bool(value) => value.to_string(),
|
||||
serde_json::Value::Null => String::new(),
|
||||
other => other.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires the remote DBX MySQL 5.7 smoke-test container"]
|
||||
async fn live_mysql57_text_protocol_select_succeeds() {
|
||||
@@ -74,6 +114,85 @@ async fn live_mysql_compatible_limited_text_protocol_query_succeeds() {
|
||||
assert!(result.rows.len() <= 100);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a writable MySQL endpoint for query-result XLSX export"]
|
||||
async fn live_mysql_query_result_export_xlsx_streams_single_query_without_duplicate_batches() {
|
||||
let host = std::env::var("DBX_LIVE_MYSQL_EXPORT_HOST").expect("DBX_LIVE_MYSQL_EXPORT_HOST");
|
||||
let port = std::env::var("DBX_LIVE_MYSQL_EXPORT_PORT").expect("DBX_LIVE_MYSQL_EXPORT_PORT").parse::<u16>().unwrap();
|
||||
let user = std::env::var("DBX_LIVE_MYSQL_EXPORT_USER").expect("DBX_LIVE_MYSQL_EXPORT_USER");
|
||||
let password = std::env::var("DBX_LIVE_MYSQL_EXPORT_PASSWORD").expect("DBX_LIVE_MYSQL_EXPORT_PASSWORD");
|
||||
let database = std::env::var("DBX_LIVE_MYSQL_EXPORT_DATABASE").expect("DBX_LIVE_MYSQL_EXPORT_DATABASE");
|
||||
let suffix = uuid::Uuid::new_v4().simple().to_string();
|
||||
let table = format!("dbx_query_export_{}", &suffix[..8]);
|
||||
let connection_id = format!("live-mysql-query-export-{suffix}");
|
||||
let config = live_mysql_query_export_config(&connection_id, &host, port, &user, &password, &database);
|
||||
let dir = std::env::temp_dir().join(format!("dbx-live-mysql-query-export-{suffix}"));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
state.configs.write().await.insert(config.id.clone(), config);
|
||||
|
||||
let values = (1..=250).map(|id| format!("({id}, 'row-{id}')")).collect::<Vec<_>>().join(", ");
|
||||
let cleanup_sql = format!("DROP TABLE IF EXISTS `{table}`");
|
||||
let create_sql = format!("CREATE TABLE `{table}` (id INT PRIMARY KEY, label VARCHAR(32) NOT NULL)");
|
||||
let insert_sql = format!("INSERT INTO `{table}` (id, label) VALUES {values}");
|
||||
let _ = execute_sql_statement(&state, &connection_id, &database, &cleanup_sql, None, None).await;
|
||||
execute_sql_statement(&state, &connection_id, &database, &create_sql, None, None)
|
||||
.await
|
||||
.expect("create live export table");
|
||||
execute_sql_statement(&state, &connection_id, &database, &insert_sql, None, None)
|
||||
.await
|
||||
.expect("insert live export rows");
|
||||
|
||||
let file_path = dir.join("result.xlsx");
|
||||
let sql = format!("SELECT id, label FROM `{table}`");
|
||||
let request = QueryResultExportRequest {
|
||||
export_id: format!("live-mysql-query-export-{suffix}"),
|
||||
connection_id: connection_id.clone(),
|
||||
database: database.clone(),
|
||||
schema: None,
|
||||
sql: sql.clone(),
|
||||
query_base_sql: sql,
|
||||
database_type: DatabaseType::Mysql,
|
||||
use_agent_cursor: false,
|
||||
file_path: file_path.to_string_lossy().to_string(),
|
||||
format: "xlsx".to_string(),
|
||||
page_size: 50,
|
||||
row_limit: None,
|
||||
total_rows: Some(250),
|
||||
timeout_secs: Some(30),
|
||||
keyset_optimization_enabled: false,
|
||||
client_session_id: None,
|
||||
execution_id: Some(format!("live-mysql-query-export-{suffix}")),
|
||||
};
|
||||
let done_seen = AtomicBool::new(false);
|
||||
let result = export_query_result_core(&state, &request, None, |progress| {
|
||||
if matches!(progress.status, ExportStatus::Done) {
|
||||
done_seen.store(true, Ordering::Relaxed);
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
let cleanup_result = execute_sql_statement(&state, &connection_id, &database, &cleanup_sql, None, None).await;
|
||||
result.expect("export MySQL query result to XLSX");
|
||||
cleanup_result.expect("cleanup live export table");
|
||||
assert!(done_seen.load(Ordering::Relaxed));
|
||||
|
||||
let parsed = parse_xlsx_file(&file_path.to_string_lossy(), 300).expect("parse exported XLSX");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
assert_eq!(parsed.columns, vec!["id", "label"]);
|
||||
assert_eq!(parsed.total_rows, 250);
|
||||
assert_eq!(parsed.rows.len(), 250);
|
||||
|
||||
let exported_ids = parsed
|
||||
.rows
|
||||
.iter()
|
||||
.map(|row| json_cell_text(&row[0]).parse::<i64>().expect("numeric id"))
|
||||
.collect::<BTreeSet<_>>();
|
||||
let expected_ids = (1..=250).collect::<BTreeSet<_>>();
|
||||
assert_eq!(exported_ids, expected_ids);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a remote MySQL endpoint"]
|
||||
async fn live_mysql_call_procedure_returns_select_result_set() {
|
||||
|
||||
Reference in new issue
Block a user