fix(mysql): stream query result exports

This commit is contained in:
t8y2 committed 2026-07-07 00:50:14 +08:00
1 parent 9cbe857a2a
commit 3609641fc8
3 files changed
+412 -9

No files matched your search

+119
View File
@@ -1,9 +1,14 @@
use std::collections::BTreeSet;
use std::sync::atomic::{AtomicBool, Ordering};
use dbx_core::connection::AppState;
use dbx_core::models::connection::{ConnectionConfig, DatabaseType};
use dbx_core::query::execute_sql_statement;
use dbx_core::query_result_export::{export_query_result_core, ExportStatus, QueryResultExportRequest};
use dbx_core::sql::{split_sql_statements_for_database, SqlFileRequest};
use dbx_core::sql_file_import::execute_sql_file_content;
use dbx_core::storage::Storage;
use dbx_core::table_import::parse_xlsx_file;
use tokio_util::sync::CancellationToken;
fn live_mysql_sql_file_config(id: &str) -> ConnectionConfig {
@@ -38,6 +43,41 @@ async fn app_state_with_config(config: ConnectionConfig) -> (AppState, std::path
(state, db_path)
}
fn live_mysql_query_export_config(
id: &str,
host: &str,
port: u16,
user: &str,
password: &str,
database: &str,
) -> ConnectionConfig {
serde_json::from_value(serde_json::json!({
"id": id,
"name": id,
"db_type": DatabaseType::Mysql,
"host": host,
"port": port,
"username": user,
"password": password,
"database": database,
"connect_timeout_secs": 10,
"query_timeout_secs": 30,
"idle_timeout_secs": 60,
"keepalive_interval_secs": 0
}))
.expect("live MySQL query export config should deserialize")
}
fn json_cell_text(value: &serde_json::Value) -> String {
match value {
serde_json::Value::String(value) => value.clone(),
serde_json::Value::Number(value) => value.to_string(),
serde_json::Value::Bool(value) => value.to_string(),
serde_json::Value::Null => String::new(),
other => other.to_string(),
}
}
#[tokio::test]
#[ignore = "requires the remote DBX MySQL 5.7 smoke-test container"]
async fn live_mysql57_text_protocol_select_succeeds() {
@@ -74,6 +114,85 @@ async fn live_mysql_compatible_limited_text_protocol_query_succeeds() {
assert!(result.rows.len() <= 100);
}
#[tokio::test]
#[ignore = "requires a writable MySQL endpoint for query-result XLSX export"]
async fn live_mysql_query_result_export_xlsx_streams_single_query_without_duplicate_batches() {
let host = std::env::var("DBX_LIVE_MYSQL_EXPORT_HOST").expect("DBX_LIVE_MYSQL_EXPORT_HOST");
let port = std::env::var("DBX_LIVE_MYSQL_EXPORT_PORT").expect("DBX_LIVE_MYSQL_EXPORT_PORT").parse::<u16>().unwrap();
let user = std::env::var("DBX_LIVE_MYSQL_EXPORT_USER").expect("DBX_LIVE_MYSQL_EXPORT_USER");
let password = std::env::var("DBX_LIVE_MYSQL_EXPORT_PASSWORD").expect("DBX_LIVE_MYSQL_EXPORT_PASSWORD");
let database = std::env::var("DBX_LIVE_MYSQL_EXPORT_DATABASE").expect("DBX_LIVE_MYSQL_EXPORT_DATABASE");
let suffix = uuid::Uuid::new_v4().simple().to_string();
let table = format!("dbx_query_export_{}", &suffix[..8]);
let connection_id = format!("live-mysql-query-export-{suffix}");
let config = live_mysql_query_export_config(&connection_id, &host, port, &user, &password, &database);
let dir = std::env::temp_dir().join(format!("dbx-live-mysql-query-export-{suffix}"));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
state.configs.write().await.insert(config.id.clone(), config);
let values = (1..=250).map(|id| format!("({id}, 'row-{id}')")).collect::<Vec<_>>().join(", ");
let cleanup_sql = format!("DROP TABLE IF EXISTS `{table}`");
let create_sql = format!("CREATE TABLE `{table}` (id INT PRIMARY KEY, label VARCHAR(32) NOT NULL)");
let insert_sql = format!("INSERT INTO `{table}` (id, label) VALUES {values}");
let _ = execute_sql_statement(&state, &connection_id, &database, &cleanup_sql, None, None).await;
execute_sql_statement(&state, &connection_id, &database, &create_sql, None, None)
.await
.expect("create live export table");
execute_sql_statement(&state, &connection_id, &database, &insert_sql, None, None)
.await
.expect("insert live export rows");
let file_path = dir.join("result.xlsx");
let sql = format!("SELECT id, label FROM `{table}`");
let request = QueryResultExportRequest {
export_id: format!("live-mysql-query-export-{suffix}"),
connection_id: connection_id.clone(),
database: database.clone(),
schema: None,
sql: sql.clone(),
query_base_sql: sql,
database_type: DatabaseType::Mysql,
use_agent_cursor: false,
file_path: file_path.to_string_lossy().to_string(),
format: "xlsx".to_string(),
page_size: 50,
row_limit: None,
total_rows: Some(250),
timeout_secs: Some(30),
keyset_optimization_enabled: false,
client_session_id: None,
execution_id: Some(format!("live-mysql-query-export-{suffix}")),
};
let done_seen = AtomicBool::new(false);
let result = export_query_result_core(&state, &request, None, |progress| {
if matches!(progress.status, ExportStatus::Done) {
done_seen.store(true, Ordering::Relaxed);
}
})
.await;
let cleanup_result = execute_sql_statement(&state, &connection_id, &database, &cleanup_sql, None, None).await;
result.expect("export MySQL query result to XLSX");
cleanup_result.expect("cleanup live export table");
assert!(done_seen.load(Ordering::Relaxed));
let parsed = parse_xlsx_file(&file_path.to_string_lossy(), 300).expect("parse exported XLSX");
let _ = std::fs::remove_dir_all(&dir);
assert_eq!(parsed.columns, vec!["id", "label"]);
assert_eq!(parsed.total_rows, 250);
assert_eq!(parsed.rows.len(), 250);
let exported_ids = parsed
.rows
.iter()
.map(|row| json_cell_text(&row[0]).parse::<i64>().expect("numeric id"))
.collect::<BTreeSet<_>>();
let expected_ids = (1..=250).collect::<BTreeSet<_>>();
assert_eq!(exported_ids, expected_ids);
}
#[tokio::test]
#[ignore = "requires a remote MySQL endpoint"]
async fn live_mysql_call_procedure_returns_select_result_set() {