diff --git a/crates/dbx-core/src/db/clickhouse_driver.rs b/crates/dbx-core/src/db/clickhouse_driver.rs index f6fba4aa94..7f07281b5c 100644 --- a/crates/dbx-core/src/db/clickhouse_driver.rs +++ b/crates/dbx-core/src/db/clickhouse_driver.rs @@ -1,10 +1,12 @@ +use futures::StreamExt; use reqwest::{Certificate, Client as HttpClient}; use serde::Deserialize; use std::fs; use std::time::{Duration, Instant}; +use tokio_util::sync::CancellationToken; use super::{http_client_builder, with_connection_timeout}; -use crate::query::MAX_ROWS; +use crate::query::{canceled_error, MAX_ROWS}; use crate::sql::starts_with_executable_sql_keyword; use crate::types::{ColumnInfo, DatabaseInfo, IndexInfo, ObjectStatistics, QueryResult, TableInfo}; @@ -100,8 +102,39 @@ enum QueryResultLimit { Limited(usize), } +enum QueryResultFormat { + JsonCompact, + JsonCompactEachRowWithNamesAndTypes, +} + +const CLICKHOUSE_STREAM_ROW_LIMIT_REACHED: &str = "__DBX_CLICKHOUSE_STREAM_ROW_LIMIT_REACHED__"; + +impl QueryResultFormat { + fn as_str(&self) -> &'static str { + match self { + QueryResultFormat::JsonCompact => "JSONCompact", + QueryResultFormat::JsonCompactEachRowWithNamesAndTypes => "JSONCompactEachRowWithNamesAndTypes", + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub enum ClickHouseQueryStreamItem { + Columns { columns: Vec, column_types: Vec }, + Row(Vec), +} + fn build_query_url(base_url: &str, database: Option<&str>, limit: QueryResultLimit) -> String { - let mut url = format!("{}/?default_format=JSONCompact", base_url); + build_query_url_with_format(base_url, database, limit, QueryResultFormat::JsonCompact) +} + +fn build_query_url_with_format( + base_url: &str, + database: Option<&str>, + limit: QueryResultLimit, + format: QueryResultFormat, +) -> String { + let mut url = format!("{}/?default_format={}", base_url, format.as_str()); if let Some(db) = database { url.push_str(&format!("&database={db}")); } @@ -111,6 +144,94 @@ fn build_query_url(base_url: &str, database: Option<&str>, limit: QueryResultLim url } +#[derive(Default)] +struct ClickHouseStreamParser { + line_index: usize, + columns: Vec, +} + +impl ClickHouseStreamParser { + fn parse_line(&mut self, line: &str) -> Result, String> { + let line = line.trim_end_matches('\r'); + if line.trim().is_empty() { + return Ok(None); + } + + match self.line_index { + 0 => { + self.columns = parse_stream_string_array(line, "column names")?; + self.line_index = 1; + Ok(None) + } + 1 => { + let column_types = parse_stream_string_array(line, "column types")?; + self.line_index = 2; + Ok(Some(ClickHouseQueryStreamItem::Columns { columns: self.columns.clone(), column_types })) + } + _ => { + let row = parse_stream_value_array(line, "row")?; + self.line_index += 1; + Ok(Some(ClickHouseQueryStreamItem::Row(row))) + } + } + } + + fn finish(&self) -> Result<(), String> { + if self.line_index == 1 { + return Err("ClickHouse stream ended before column types were received".to_string()); + } + Ok(()) + } +} + +fn parse_stream_string_array(line: &str, label: &str) -> Result, String> { + parse_stream_value_array(line, label).map(|values| { + values + .into_iter() + .map(|value| match value { + serde_json::Value::String(value) => value, + other => other.to_string(), + }) + .collect() + }) +} + +fn parse_stream_value_array(line: &str, label: &str) -> Result, String> { + serde_json::from_str(line).map_err(|e| format!("ClickHouse stream parse error in {label}: {e}")) +} + +fn process_stream_buffer( + buffer: &mut Vec, + parser: &mut ClickHouseStreamParser, + on_item: &mut impl FnMut(ClickHouseQueryStreamItem) -> Result<(), String>, +) -> Result<(), String> { + while let Some(newline) = buffer.iter().position(|byte| *byte == b'\n') { + let line_bytes: Vec = buffer.drain(..=newline).collect(); + let line_bytes = &line_bytes[..line_bytes.len().saturating_sub(1)]; + let line = + std::str::from_utf8(line_bytes).map_err(|e| format!("ClickHouse stream contains invalid UTF-8: {e}"))?; + if let Some(item) = parser.parse_line(line)? { + on_item(item)?; + } + } + Ok(()) +} + +fn process_stream_remainder( + buffer: &mut Vec, + parser: &mut ClickHouseStreamParser, + on_item: &mut impl FnMut(ClickHouseQueryStreamItem) -> Result<(), String>, +) -> Result<(), String> { + if !buffer.is_empty() { + let line = std::str::from_utf8(buffer).map_err(|e| format!("ClickHouse stream contains invalid UTF-8: {e}"))?; + if let Some(item) = parser.parse_line(line)? { + on_item(item)?; + } + buffer.clear(); + } + parser.finish() +} + fn build_request(client: &ChClient, req: reqwest::RequestBuilder) -> reqwest::RequestBuilder { match (&client.username, &client.password) { (Some(u), Some(p)) if !u.is_empty() => req.basic_auth(u, Some(p)), @@ -142,6 +263,75 @@ async fn ch_query_with_limit( resp.json::().await.map_err(|e| format!("ClickHouse parse error: {e}")) } +pub async fn stream_query_with_max_rows( + client: &ChClient, + database: &str, + sql: &str, + max_rows: Option, + cancel_token: Option, + mut on_item: impl FnMut(ClickHouseQueryStreamItem) -> Result<(), String>, +) -> Result<(), String> { + let max_rows = max_rows.map(|max_rows| max_rows.max(1)); + let limit = max_rows.map(QueryResultLimit::Limited).unwrap_or(QueryResultLimit::Unlimited); + let url = build_query_url_with_format( + &client.base_url, + Some(database), + limit, + QueryResultFormat::JsonCompactEachRowWithNamesAndTypes, + ); + log::info!("[clickhouse] stream query url={url} user={:?} has_pass={}", client.username, client.password.is_some()); + let req = build_request(client, client.http.post(&url).body(sql.to_string())); + let resp = req.send().await.map_err(|e| format!("ClickHouse request failed: {e}"))?; + log::info!("[clickhouse] stream response status={}", resp.status()); + if !resp.status().is_success() { + let body = resp.text().await.unwrap_or_default(); + log::error!("[clickhouse] stream error body: {body}"); + return Err(format!("ClickHouse error: {body}")); + } + + let mut stream = resp.bytes_stream(); + let mut buffer = Vec::new(); + let mut parser = ClickHouseStreamParser::default(); + let mut rows_seen = 0usize; + let mut emit_item = |item: ClickHouseQueryStreamItem| -> Result<(), String> { + if matches!(item, ClickHouseQueryStreamItem::Row(_)) { + rows_seen = rows_seen.saturating_add(1); + } + on_item(item)?; + if max_rows.is_some_and(|max_rows| rows_seen >= max_rows) { + return Err(CLICKHOUSE_STREAM_ROW_LIMIT_REACHED.to_string()); + } + Ok(()) + }; + + loop { + let chunk = match cancel_token.as_ref() { + Some(token) => { + tokio::select! { + biased; + _ = token.cancelled() => return Err(canceled_error()), + chunk = stream.next() => chunk, + } + } + None => stream.next().await, + }; + let Some(chunk) = chunk else { + break; + }; + let chunk = chunk.map_err(|e| format!("ClickHouse stream read failed: {e}"))?; + buffer.extend_from_slice(&chunk); + match process_stream_buffer(&mut buffer, &mut parser, &mut emit_item) { + Err(error) if error == CLICKHOUSE_STREAM_ROW_LIMIT_REACHED => return Ok(()), + result => result?, + } + } + + match process_stream_remainder(&mut buffer, &mut parser, &mut emit_item) { + Err(error) if error == CLICKHOUSE_STREAM_ROW_LIMIT_REACHED => Ok(()), + result => result, + } +} + fn query_result_row_limit(max_rows: Option) -> usize { max_rows.unwrap_or(MAX_ROWS).max(1) } @@ -449,6 +639,52 @@ mod tests { ); } + #[test] + fn stream_query_url_uses_line_delimited_names_and_types_format() { + let url = build_query_url_with_format( + "http://localhost:8123", + Some("analytics"), + QueryResultLimit::Limited(500), + QueryResultFormat::JsonCompactEachRowWithNamesAndTypes, + ); + + assert_eq!( + url, + "http://localhost:8123/?default_format=JSONCompactEachRowWithNamesAndTypes&database=analytics&max_result_rows=500&result_overflow_mode=break" + ); + } + + #[test] + fn stream_parser_reads_columns_types_and_rows_across_chunks() { + let mut buffer = b"[\"id\",\"name\"]\n[\"UInt64\",\"String\"]\n[1,\"Ada\"]\n[2,\"Linus\"".to_vec(); + let mut parser = ClickHouseStreamParser::default(); + let mut items = Vec::new(); + + process_stream_buffer(&mut buffer, &mut parser, &mut |item| { + items.push(item); + Ok(()) + }) + .unwrap(); + buffer.extend_from_slice(b"]\n"); + process_stream_remainder(&mut buffer, &mut parser, &mut |item| { + items.push(item); + Ok(()) + }) + .unwrap(); + + assert_eq!( + items, + vec![ + ClickHouseQueryStreamItem::Columns { + columns: vec!["id".to_string(), "name".to_string()], + column_types: vec!["UInt64".to_string(), "String".to_string()], + }, + ClickHouseQueryStreamItem::Row(vec![serde_json::json!(1), serde_json::json!("Ada")]), + ClickHouseQueryStreamItem::Row(vec![serde_json::json!(2), serde_json::json!("Linus")]), + ] + ); + } + #[test] fn limited_query_result_truncates_extra_probe_row() { let result = ChJsonResult { diff --git a/crates/dbx-core/src/query_result_export.rs b/crates/dbx-core/src/query_result_export.rs index 3d889d222d..1fc233e860 100644 --- a/crates/dbx-core/src/query_result_export.rs +++ b/crates/dbx-core/src/query_result_export.rs @@ -333,6 +333,12 @@ async fn export_query_result_core_inner( return Ok(()); } + // ClickHouse HTTP pagination is unsafe for unsorted result sets; stream one + // response so large exports preserve the server's single execution order. + if try_export_clickhouse_query_result_stream(state, request, &format, cancel_token.clone(), on_progress).await? { + return Ok(()); + } + let mut csv_file = if format == "csv" { Some(BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?)) } else { @@ -950,6 +956,174 @@ async fn try_export_mysql_query_result_stream( Ok(true) } +async fn try_export_clickhouse_query_result_stream( + state: &AppState, + request: &QueryResultExportRequest, + format: &str, + cancel_token: Option, + on_progress: &impl Fn(TableExportProgress), +) -> Result { + if request.database_type != DatabaseType::ClickHouse + || request.use_agent_cursor + || !crate::sql::starts_with_executable_sql_keyword( + &request.sql, + &["SELECT", "SHOW", "DESCRIBE", "EXPLAIN", "WITH"], + ) + { + return Ok(false); + } + + let database = request.database.trim(); + let pool_key = if database.is_empty() { + state.get_or_create_pool_for_session(&request.connection_id, None, request.client_session_id.as_deref()).await? + } else { + state + .get_or_create_pool_for_session( + &request.connection_id, + Some(database), + request.client_session_id.as_deref(), + ) + .await? + }; + let connections = state.connections.read().await; + let Some(client) = connections.get(&pool_key).and_then(|pool| match pool { + PoolKind::ClickHouse(client) => Some(client.clone()), + _ => None, + }) else { + return Ok(false); + }; + drop(connections); + + if let Some(execution_id) = request.execution_id.as_deref() { + state.running_queries.set_pool_key(execution_id, pool_key.clone()); + } + state.touch_pool_activity(&pool_key).await; + let _activity_touch = state.pool_activity_touch(&pool_key); + + let xlsx_hard_limit_active = xlsx_hard_limit_active(format, request); + let row_limit = effective_row_limit(format, request); + let stream_row_limit = + if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit }; + let progress_row_interval = request.page_size.max(1) as u64; + let mut columns: Vec = Vec::new(); + let mut rows_exported = 0_u64; + let mut last_progress_rows = 0_u64; + let mut last_progress_at = Instant::now(); + let mut csv_file = if format == "csv" { + let mut file = + BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?); + file.write_all(b"\xEF\xBB\xBF").map_err(|e| format!("Failed to write BOM: {e}"))?; + Some(file) + } else { + None + }; + let mut xlsx = None; + let clickhouse_database = if database.is_empty() { "default" } else { database }; + + let stream_future = crate::db::clickhouse_driver::stream_query_with_max_rows( + &client, + clickhouse_database, + &request.sql, + stream_row_limit, + cancel_token.clone(), + |item| { + match item { + crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Columns { + columns: stream_columns, .. + } => { + columns = stream_columns; + if let Some(file) = csv_file.as_mut() { + let csv = format_query_result_csv(&columns, &[]); + let header = csv.strip_suffix('\n').unwrap_or(&csv); + file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?; + } else { + let xlsx_file = + File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?; + xlsx = + Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?); + } + } + crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Row(row) => { + if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS { + return Err(XLSX_ROW_LIMIT_ERROR.to_string()); + } + if let Some(file) = csv_file.as_mut() { + let rows_csv = format_query_result_csv_rows(std::slice::from_ref(&row)); + write!(file, "\n{rows_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?; + } else if let Some(writer) = xlsx.as_mut() { + writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + } else { + let xlsx_file = + File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?; + xlsx = + Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?); + if let Some(writer) = xlsx.as_mut() { + writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + } + } + rows_exported += 1; + let now = Instant::now(); + if should_emit_stream_progress( + rows_exported, + last_progress_rows, + progress_row_interval, + now.duration_since(last_progress_at), + ) { + on_progress(progress(request, rows_exported, ExportStatus::Running, None)); + last_progress_rows = rows_exported; + last_progress_at = now; + } + } + } + Ok(()) + }, + ); + let stream_result = match query_export_timeout(request.timeout_secs) { + Some(timeout) => match tokio::time::timeout(timeout, stream_future).await { + Ok(result) => result, + Err(_) => Err(format!("Query timed out after {} seconds", timeout.as_secs())), + }, + None => stream_future.await, + }; + + if let Err(error) = stream_result { + if error == QUERY_CANCELED + || cancel_token.as_ref().is_some_and(|token| token.is_cancelled()) + || is_export_cancelled(&request.export_id).await + { + on_progress(progress( + request, + rows_exported, + ExportStatus::Cancelled, + Some("Export cancelled".to_string()), + )); + return Ok(true); + } + return Err(error); + } + + if rows_exported != last_progress_rows { + on_progress(progress(request, rows_exported, ExportStatus::Running, None)); + } + on_progress(progress(request, rows_exported, ExportStatus::Writing, None)); + if let Some(file) = csv_file.as_mut() { + file.flush().map_err(|e| format!("Failed to flush CSV file: {e}"))?; + } + if let Some(writer) = xlsx { + let mut buf = + finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?; + buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?; + } else if format == "xlsx" { + let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?; + let writer = start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?; + let mut buf = + finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?; + buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?; + } + on_progress(progress(request, rows_exported, ExportStatus::Done, None)); + Ok(true) +} + async fn try_export_sqlserver_query_result_stream( state: &AppState, request: &QueryResultExportRequest, diff --git a/crates/dbx-core/tests/live_clickhouse_query_result_export.rs b/crates/dbx-core/tests/live_clickhouse_query_result_export.rs new file mode 100644 index 0000000000..beb693ae1c --- /dev/null +++ b/crates/dbx-core/tests/live_clickhouse_query_result_export.rs @@ -0,0 +1,121 @@ +use std::collections::BTreeSet; +use std::sync::atomic::{AtomicBool, Ordering}; + +use dbx_core::connection::AppState; +use dbx_core::models::connection::{ConnectionConfig, DatabaseType}; +use dbx_core::query_result_export::{export_query_result_core, ExportStatus, QueryResultExportRequest}; +use dbx_core::storage::Storage; +use dbx_core::table_import::parse_xlsx_file; + +fn live_clickhouse_config( + id: &str, + host: &str, + port: u16, + user: &str, + password: &str, + database: &str, +) -> ConnectionConfig { + serde_json::from_value(serde_json::json!({ + "id": id, + "name": id, + "db_type": DatabaseType::ClickHouse, + "host": host, + "port": port, + "username": user, + "password": password, + "database": database, + "connect_timeout_secs": 10, + "query_timeout_secs": 30, + "idle_timeout_secs": 60, + "keepalive_interval_secs": 0 + })) + .expect("live ClickHouse config should deserialize") +} + +fn json_cell_text(value: &serde_json::Value) -> String { + match value { + serde_json::Value::String(value) => value.clone(), + serde_json::Value::Number(value) => value.to_string(), + serde_json::Value::Bool(value) => value.to_string(), + serde_json::Value::Null => String::new(), + other => other.to_string(), + } +} + +fn json_cell_i64(value: &serde_json::Value) -> i64 { + if let Some(value) = value.as_i64() { + return value; + } + if let Some(value) = value.as_u64() { + return i64::try_from(value).expect("u64 id fits in i64"); + } + let text = json_cell_text(value); + text.parse::().or_else(|_| text.parse::().map(|value| value as i64)).expect("numeric id") +} + +#[tokio::test] +#[ignore = "requires DBX_LIVE_CLICKHOUSE_* env vars pointing at a ClickHouse HTTP endpoint"] +async fn live_clickhouse_query_result_export_xlsx_streams_random_order_query_once() { + let host = std::env::var("DBX_LIVE_CLICKHOUSE_HOST").expect("DBX_LIVE_CLICKHOUSE_HOST"); + let port = std::env::var("DBX_LIVE_CLICKHOUSE_PORT").expect("DBX_LIVE_CLICKHOUSE_PORT").parse::().unwrap(); + let user = std::env::var("DBX_LIVE_CLICKHOUSE_USER").expect("DBX_LIVE_CLICKHOUSE_USER"); + let password = std::env::var("DBX_LIVE_CLICKHOUSE_PASSWORD").expect("DBX_LIVE_CLICKHOUSE_PASSWORD"); + let database = std::env::var("DBX_LIVE_CLICKHOUSE_DATABASE").expect("DBX_LIVE_CLICKHOUSE_DATABASE"); + let suffix = uuid::Uuid::new_v4().simple().to_string(); + let connection_id = format!("live-clickhouse-query-export-{suffix}"); + let config = live_clickhouse_config(&connection_id, &host, port, &user, &password, &database); + let dir = std::env::temp_dir().join(format!("dbx-live-clickhouse-query-export-{suffix}")); + std::fs::create_dir_all(&dir).unwrap(); + let storage = Storage::open(&dir.join("storage.db")).await.unwrap(); + let state = AppState::new(storage); + state.configs.write().await.insert(config.id.clone(), config); + + let file_path = dir.join("result.xlsx"); + let sql = " + SELECT + number + 1 AS id, + concat('row-', toString(number + 1)) AS label + FROM numbers(2050) + ORDER BY rand() + " + .to_string(); + let request = QueryResultExportRequest { + export_id: format!("live-clickhouse-query-export-{suffix}"), + connection_id: connection_id.clone(), + database: database.clone(), + schema: None, + sql: sql.clone(), + query_base_sql: sql, + database_type: DatabaseType::ClickHouse, + use_agent_cursor: false, + file_path: file_path.to_string_lossy().to_string(), + format: "xlsx".to_string(), + page_size: 100, + row_limit: None, + total_rows: Some(2050), + timeout_secs: Some(30), + keyset_optimization_enabled: false, + client_session_id: None, + execution_id: Some(format!("live-clickhouse-query-export-{suffix}")), + }; + let done_seen = AtomicBool::new(false); + let result = export_query_result_core(&state, &request, None, |progress| { + if matches!(progress.status, ExportStatus::Done) { + done_seen.store(true, Ordering::Relaxed); + } + }) + .await; + + result.expect("export ClickHouse query result to XLSX without per-page re-execution"); + assert!(done_seen.load(Ordering::Relaxed)); + + let parsed = parse_xlsx_file(&file_path.to_string_lossy(), 3000).expect("parse exported XLSX"); + let _ = std::fs::remove_dir_all(&dir); + assert_eq!(parsed.columns, vec!["id", "label"]); + assert_eq!(parsed.total_rows, 2050); + assert_eq!(parsed.rows.len(), 2050); + + let exported_ids = parsed.rows.iter().map(|row| json_cell_i64(&row[0])).collect::>(); + let expected_ids = (1..=2050).collect::>(); + assert_eq!(exported_ids, expected_ids); +}