fix(clickhouse): stream query result exports

This commit is contained in:
t8y2 committed 2026-07-09 13:49:59 +08:00
1 parent e87e31752a
commit 315c6e235b
3 files changed
+533 -2

No files matched your search

+238 -2
View File
@@ -1,10 +1,12 @@
use futures::StreamExt;
use reqwest::{Certificate, Client as HttpClient}; use reqwest::{Certificate, Client as HttpClient};
use serde::Deserialize; use serde::Deserialize;
use std::fs; use std::fs;
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
use tokio_util::sync::CancellationToken;
use super::{http_client_builder, with_connection_timeout}; use super::{http_client_builder, with_connection_timeout};
use crate::query::MAX_ROWS; use crate::query::{canceled_error, MAX_ROWS};
use crate::sql::starts_with_executable_sql_keyword; use crate::sql::starts_with_executable_sql_keyword;
use crate::types::{ColumnInfo, DatabaseInfo, IndexInfo, ObjectStatistics, QueryResult, TableInfo}; use crate::types::{ColumnInfo, DatabaseInfo, IndexInfo, ObjectStatistics, QueryResult, TableInfo};
@@ -100,8 +102,39 @@ enum QueryResultLimit {
Limited(usize), Limited(usize),
} }
enum QueryResultFormat {
JsonCompact,
JsonCompactEachRowWithNamesAndTypes,
}
const CLICKHOUSE_STREAM_ROW_LIMIT_REACHED: &str = "__DBX_CLICKHOUSE_STREAM_ROW_LIMIT_REACHED__";
impl QueryResultFormat {
fn as_str(&self) -> &'static str {
match self {
QueryResultFormat::JsonCompact => "JSONCompact",
QueryResultFormat::JsonCompactEachRowWithNamesAndTypes => "JSONCompactEachRowWithNamesAndTypes",
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum ClickHouseQueryStreamItem {
Columns { columns: Vec<String>, column_types: Vec<String> },
Row(Vec<serde_json::Value>),
}
fn build_query_url(base_url: &str, database: Option<&str>, limit: QueryResultLimit) -> String { fn build_query_url(base_url: &str, database: Option<&str>, limit: QueryResultLimit) -> String {
let mut url = format!("{}/?default_format=JSONCompact", base_url); build_query_url_with_format(base_url, database, limit, QueryResultFormat::JsonCompact)
}
fn build_query_url_with_format(
base_url: &str,
database: Option<&str>,
limit: QueryResultLimit,
format: QueryResultFormat,
) -> String {
let mut url = format!("{}/?default_format={}", base_url, format.as_str());
if let Some(db) = database { if let Some(db) = database {
url.push_str(&format!("&database={db}")); url.push_str(&format!("&database={db}"));
} }
@@ -111,6 +144,94 @@ fn build_query_url(base_url: &str, database: Option<&str>, limit: QueryResultLim
url url
} }
#[derive(Default)]
struct ClickHouseStreamParser {
line_index: usize,
columns: Vec<String>,
}
impl ClickHouseStreamParser {
fn parse_line(&mut self, line: &str) -> Result<Option<ClickHouseQueryStreamItem>, String> {
let line = line.trim_end_matches('\r');
if line.trim().is_empty() {
return Ok(None);
}
match self.line_index {
0 => {
self.columns = parse_stream_string_array(line, "column names")?;
self.line_index = 1;
Ok(None)
}
1 => {
let column_types = parse_stream_string_array(line, "column types")?;
self.line_index = 2;
Ok(Some(ClickHouseQueryStreamItem::Columns { columns: self.columns.clone(), column_types }))
}
_ => {
let row = parse_stream_value_array(line, "row")?;
self.line_index += 1;
Ok(Some(ClickHouseQueryStreamItem::Row(row)))
}
}
}
fn finish(&self) -> Result<(), String> {
if self.line_index == 1 {
return Err("ClickHouse stream ended before column types were received".to_string());
}
Ok(())
}
}
fn parse_stream_string_array(line: &str, label: &str) -> Result<Vec<String>, String> {
parse_stream_value_array(line, label).map(|values| {
values
.into_iter()
.map(|value| match value {
serde_json::Value::String(value) => value,
other => other.to_string(),
})
.collect()
})
}
fn parse_stream_value_array(line: &str, label: &str) -> Result<Vec<serde_json::Value>, String> {
serde_json::from_str(line).map_err(|e| format!("ClickHouse stream parse error in {label}: {e}"))
}
fn process_stream_buffer(
buffer: &mut Vec<u8>,
parser: &mut ClickHouseStreamParser,
on_item: &mut impl FnMut(ClickHouseQueryStreamItem) -> Result<(), String>,
) -> Result<(), String> {
while let Some(newline) = buffer.iter().position(|byte| *byte == b'\n') {
let line_bytes: Vec<u8> = buffer.drain(..=newline).collect();
let line_bytes = &line_bytes[..line_bytes.len().saturating_sub(1)];
let line =
std::str::from_utf8(line_bytes).map_err(|e| format!("ClickHouse stream contains invalid UTF-8: {e}"))?;
if let Some(item) = parser.parse_line(line)? {
on_item(item)?;
}
}
Ok(())
}
fn process_stream_remainder(
buffer: &mut Vec<u8>,
parser: &mut ClickHouseStreamParser,
on_item: &mut impl FnMut(ClickHouseQueryStreamItem) -> Result<(), String>,
) -> Result<(), String> {
if !buffer.is_empty() {
let line = std::str::from_utf8(buffer).map_err(|e| format!("ClickHouse stream contains invalid UTF-8: {e}"))?;
if let Some(item) = parser.parse_line(line)? {
on_item(item)?;
}
buffer.clear();
}
parser.finish()
}
fn build_request(client: &ChClient, req: reqwest::RequestBuilder) -> reqwest::RequestBuilder { fn build_request(client: &ChClient, req: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
match (&client.username, &client.password) { match (&client.username, &client.password) {
(Some(u), Some(p)) if !u.is_empty() => req.basic_auth(u, Some(p)), (Some(u), Some(p)) if !u.is_empty() => req.basic_auth(u, Some(p)),
@@ -142,6 +263,75 @@ async fn ch_query_with_limit(
resp.json::<ChJsonResult>().await.map_err(|e| format!("ClickHouse parse error: {e}")) resp.json::<ChJsonResult>().await.map_err(|e| format!("ClickHouse parse error: {e}"))
} }
pub async fn stream_query_with_max_rows(
client: &ChClient,
database: &str,
sql: &str,
max_rows: Option<usize>,
cancel_token: Option<CancellationToken>,
mut on_item: impl FnMut(ClickHouseQueryStreamItem) -> Result<(), String>,
) -> Result<(), String> {
let max_rows = max_rows.map(|max_rows| max_rows.max(1));
let limit = max_rows.map(QueryResultLimit::Limited).unwrap_or(QueryResultLimit::Unlimited);
let url = build_query_url_with_format(
&client.base_url,
Some(database),
limit,
QueryResultFormat::JsonCompactEachRowWithNamesAndTypes,
);
log::info!("[clickhouse] stream query url={url} user={:?} has_pass={}", client.username, client.password.is_some());
let req = build_request(client, client.http.post(&url).body(sql.to_string()));
let resp = req.send().await.map_err(|e| format!("ClickHouse request failed: {e}"))?;
log::info!("[clickhouse] stream response status={}", resp.status());
if !resp.status().is_success() {
let body = resp.text().await.unwrap_or_default();
log::error!("[clickhouse] stream error body: {body}");
return Err(format!("ClickHouse error: {body}"));
}
let mut stream = resp.bytes_stream();
let mut buffer = Vec::new();
let mut parser = ClickHouseStreamParser::default();
let mut rows_seen = 0usize;
let mut emit_item = |item: ClickHouseQueryStreamItem| -> Result<(), String> {
if matches!(item, ClickHouseQueryStreamItem::Row(_)) {
rows_seen = rows_seen.saturating_add(1);
}
on_item(item)?;
if max_rows.is_some_and(|max_rows| rows_seen >= max_rows) {
return Err(CLICKHOUSE_STREAM_ROW_LIMIT_REACHED.to_string());
}
Ok(())
};
loop {
let chunk = match cancel_token.as_ref() {
Some(token) => {
tokio::select! {
biased;
_ = token.cancelled() => return Err(canceled_error()),
chunk = stream.next() => chunk,
}
}
None => stream.next().await,
};
let Some(chunk) = chunk else {
break;
};
let chunk = chunk.map_err(|e| format!("ClickHouse stream read failed: {e}"))?;
buffer.extend_from_slice(&chunk);
match process_stream_buffer(&mut buffer, &mut parser, &mut emit_item) {
Err(error) if error == CLICKHOUSE_STREAM_ROW_LIMIT_REACHED => return Ok(()),
result => result?,
}
}
match process_stream_remainder(&mut buffer, &mut parser, &mut emit_item) {
Err(error) if error == CLICKHOUSE_STREAM_ROW_LIMIT_REACHED => Ok(()),
result => result,
}
}
fn query_result_row_limit(max_rows: Option<usize>) -> usize { fn query_result_row_limit(max_rows: Option<usize>) -> usize {
max_rows.unwrap_or(MAX_ROWS).max(1) max_rows.unwrap_or(MAX_ROWS).max(1)
} }
@@ -449,6 +639,52 @@ mod tests {
); );
} }
#[test]
fn stream_query_url_uses_line_delimited_names_and_types_format() {
let url = build_query_url_with_format(
"http://localhost:8123",
Some("analytics"),
QueryResultLimit::Limited(500),
QueryResultFormat::JsonCompactEachRowWithNamesAndTypes,
);
assert_eq!(
url,
"http://localhost:8123/?default_format=JSONCompactEachRowWithNamesAndTypes&database=analytics&max_result_rows=500&result_overflow_mode=break"
);
}
#[test]
fn stream_parser_reads_columns_types_and_rows_across_chunks() {
let mut buffer = b"[\"id\",\"name\"]\n[\"UInt64\",\"String\"]\n[1,\"Ada\"]\n[2,\"Linus\"".to_vec();
let mut parser = ClickHouseStreamParser::default();
let mut items = Vec::new();
process_stream_buffer(&mut buffer, &mut parser, &mut |item| {
items.push(item);
Ok(())
})
.unwrap();
buffer.extend_from_slice(b"]\n");
process_stream_remainder(&mut buffer, &mut parser, &mut |item| {
items.push(item);
Ok(())
})
.unwrap();
assert_eq!(
items,
vec![
ClickHouseQueryStreamItem::Columns {
columns: vec!["id".to_string(), "name".to_string()],
column_types: vec!["UInt64".to_string(), "String".to_string()],
},
ClickHouseQueryStreamItem::Row(vec![serde_json::json!(1), serde_json::json!("Ada")]),
ClickHouseQueryStreamItem::Row(vec![serde_json::json!(2), serde_json::json!("Linus")]),
]
);
}
#[test] #[test]
fn limited_query_result_truncates_extra_probe_row() { fn limited_query_result_truncates_extra_probe_row() {
let result = ChJsonResult { let result = ChJsonResult {
+174
View File
@@ -333,6 +333,12 @@ async fn export_query_result_core_inner(
return Ok(()); return Ok(());
} }
// ClickHouse HTTP pagination is unsafe for unsorted result sets; stream one
// response so large exports preserve the server's single execution order.
if try_export_clickhouse_query_result_stream(state, request, &format, cancel_token.clone(), on_progress).await? {
return Ok(());
}
let mut csv_file = if format == "csv" { let mut csv_file = if format == "csv" {
Some(BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?)) Some(BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?))
} else { } else {
@@ -950,6 +956,174 @@ async fn try_export_mysql_query_result_stream(
Ok(true) Ok(true)
} }
async fn try_export_clickhouse_query_result_stream(
state: &AppState,
request: &QueryResultExportRequest,
format: &str,
cancel_token: Option<CancellationToken>,
on_progress: &impl Fn(TableExportProgress),
) -> Result<bool, String> {
if request.database_type != DatabaseType::ClickHouse
|| request.use_agent_cursor
|| !crate::sql::starts_with_executable_sql_keyword(
&request.sql,
&["SELECT", "SHOW", "DESCRIBE", "EXPLAIN", "WITH"],
)
{
return Ok(false);
}
let database = request.database.trim();
let pool_key = if database.is_empty() {
state.get_or_create_pool_for_session(&request.connection_id, None, request.client_session_id.as_deref()).await?
} else {
state
.get_or_create_pool_for_session(
&request.connection_id,
Some(database),
request.client_session_id.as_deref(),
)
.await?
};
let connections = state.connections.read().await;
let Some(client) = connections.get(&pool_key).and_then(|pool| match pool {
PoolKind::ClickHouse(client) => Some(client.clone()),
_ => None,
}) else {
return Ok(false);
};
drop(connections);
if let Some(execution_id) = request.execution_id.as_deref() {
state.running_queries.set_pool_key(execution_id, pool_key.clone());
}
state.touch_pool_activity(&pool_key).await;
let _activity_touch = state.pool_activity_touch(&pool_key);
let xlsx_hard_limit_active = xlsx_hard_limit_active(format, request);
let row_limit = effective_row_limit(format, request);
let stream_row_limit =
if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit };
let progress_row_interval = request.page_size.max(1) as u64;
let mut columns: Vec<String> = Vec::new();
let mut rows_exported = 0_u64;
let mut last_progress_rows = 0_u64;
let mut last_progress_at = Instant::now();
let mut csv_file = if format == "csv" {
let mut file =
BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?);
file.write_all(b"\xEF\xBB\xBF").map_err(|e| format!("Failed to write BOM: {e}"))?;
Some(file)
} else {
None
};
let mut xlsx = None;
let clickhouse_database = if database.is_empty() { "default" } else { database };
let stream_future = crate::db::clickhouse_driver::stream_query_with_max_rows(
&client,
clickhouse_database,
&request.sql,
stream_row_limit,
cancel_token.clone(),
|item| {
match item {
crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Columns {
columns: stream_columns, ..
} => {
columns = stream_columns;
if let Some(file) = csv_file.as_mut() {
let csv = format_query_result_csv(&columns, &[]);
let header = csv.strip_suffix('\n').unwrap_or(&csv);
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
} else {
let xlsx_file =
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
xlsx =
Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
}
}
crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Row(row) => {
if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS {
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
}
if let Some(file) = csv_file.as_mut() {
let rows_csv = format_query_result_csv_rows(std::slice::from_ref(&row));
write!(file, "\n{rows_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?;
} else if let Some(writer) = xlsx.as_mut() {
writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
} else {
let xlsx_file =
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
xlsx =
Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
if let Some(writer) = xlsx.as_mut() {
writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
}
}
rows_exported += 1;
let now = Instant::now();
if should_emit_stream_progress(
rows_exported,
last_progress_rows,
progress_row_interval,
now.duration_since(last_progress_at),
) {
on_progress(progress(request, rows_exported, ExportStatus::Running, None));
last_progress_rows = rows_exported;
last_progress_at = now;
}
}
}
Ok(())
},
);
let stream_result = match query_export_timeout(request.timeout_secs) {
Some(timeout) => match tokio::time::timeout(timeout, stream_future).await {
Ok(result) => result,
Err(_) => Err(format!("Query timed out after {} seconds", timeout.as_secs())),
},
None => stream_future.await,
};
if let Err(error) = stream_result {
if error == QUERY_CANCELED
|| cancel_token.as_ref().is_some_and(|token| token.is_cancelled())
|| is_export_cancelled(&request.export_id).await
{
on_progress(progress(
request,
rows_exported,
ExportStatus::Cancelled,
Some("Export cancelled".to_string()),
));
return Ok(true);
}
return Err(error);
}
if rows_exported != last_progress_rows {
on_progress(progress(request, rows_exported, ExportStatus::Running, None));
}
on_progress(progress(request, rows_exported, ExportStatus::Writing, None));
if let Some(file) = csv_file.as_mut() {
file.flush().map_err(|e| format!("Failed to flush CSV file: {e}"))?;
}
if let Some(writer) = xlsx {
let mut buf =
finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?;
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
} else if format == "xlsx" {
let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
let writer = start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?;
let mut buf =
finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?;
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
}
on_progress(progress(request, rows_exported, ExportStatus::Done, None));
Ok(true)
}
async fn try_export_sqlserver_query_result_stream( async fn try_export_sqlserver_query_result_stream(
state: &AppState, state: &AppState,
request: &QueryResultExportRequest, request: &QueryResultExportRequest,
@@ -0,0 +1,121 @@
use std::collections::BTreeSet;
use std::sync::atomic::{AtomicBool, Ordering};
use dbx_core::connection::AppState;
use dbx_core::models::connection::{ConnectionConfig, DatabaseType};
use dbx_core::query_result_export::{export_query_result_core, ExportStatus, QueryResultExportRequest};
use dbx_core::storage::Storage;
use dbx_core::table_import::parse_xlsx_file;
fn live_clickhouse_config(
id: &str,
host: &str,
port: u16,
user: &str,
password: &str,
database: &str,
) -> ConnectionConfig {
serde_json::from_value(serde_json::json!({
"id": id,
"name": id,
"db_type": DatabaseType::ClickHouse,
"host": host,
"port": port,
"username": user,
"password": password,
"database": database,
"connect_timeout_secs": 10,
"query_timeout_secs": 30,
"idle_timeout_secs": 60,
"keepalive_interval_secs": 0
}))
.expect("live ClickHouse config should deserialize")
}
fn json_cell_text(value: &serde_json::Value) -> String {
match value {
serde_json::Value::String(value) => value.clone(),
serde_json::Value::Number(value) => value.to_string(),
serde_json::Value::Bool(value) => value.to_string(),
serde_json::Value::Null => String::new(),
other => other.to_string(),
}
}
fn json_cell_i64(value: &serde_json::Value) -> i64 {
if let Some(value) = value.as_i64() {
return value;
}
if let Some(value) = value.as_u64() {
return i64::try_from(value).expect("u64 id fits in i64");
}
let text = json_cell_text(value);
text.parse::<i64>().or_else(|_| text.parse::<f64>().map(|value| value as i64)).expect("numeric id")
}
#[tokio::test]
#[ignore = "requires DBX_LIVE_CLICKHOUSE_* env vars pointing at a ClickHouse HTTP endpoint"]
async fn live_clickhouse_query_result_export_xlsx_streams_random_order_query_once() {
let host = std::env::var("DBX_LIVE_CLICKHOUSE_HOST").expect("DBX_LIVE_CLICKHOUSE_HOST");
let port = std::env::var("DBX_LIVE_CLICKHOUSE_PORT").expect("DBX_LIVE_CLICKHOUSE_PORT").parse::<u16>().unwrap();
let user = std::env::var("DBX_LIVE_CLICKHOUSE_USER").expect("DBX_LIVE_CLICKHOUSE_USER");
let password = std::env::var("DBX_LIVE_CLICKHOUSE_PASSWORD").expect("DBX_LIVE_CLICKHOUSE_PASSWORD");
let database = std::env::var("DBX_LIVE_CLICKHOUSE_DATABASE").expect("DBX_LIVE_CLICKHOUSE_DATABASE");
let suffix = uuid::Uuid::new_v4().simple().to_string();
let connection_id = format!("live-clickhouse-query-export-{suffix}");
let config = live_clickhouse_config(&connection_id, &host, port, &user, &password, &database);
let dir = std::env::temp_dir().join(format!("dbx-live-clickhouse-query-export-{suffix}"));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
state.configs.write().await.insert(config.id.clone(), config);
let file_path = dir.join("result.xlsx");
let sql = "
SELECT
number + 1 AS id,
concat('row-', toString(number + 1)) AS label
FROM numbers(2050)
ORDER BY rand()
"
.to_string();
let request = QueryResultExportRequest {
export_id: format!("live-clickhouse-query-export-{suffix}"),
connection_id: connection_id.clone(),
database: database.clone(),
schema: None,
sql: sql.clone(),
query_base_sql: sql,
database_type: DatabaseType::ClickHouse,
use_agent_cursor: false,
file_path: file_path.to_string_lossy().to_string(),
format: "xlsx".to_string(),
page_size: 100,
row_limit: None,
total_rows: Some(2050),
timeout_secs: Some(30),
keyset_optimization_enabled: false,
client_session_id: None,
execution_id: Some(format!("live-clickhouse-query-export-{suffix}")),
};
let done_seen = AtomicBool::new(false);
let result = export_query_result_core(&state, &request, None, |progress| {
if matches!(progress.status, ExportStatus::Done) {
done_seen.store(true, Ordering::Relaxed);
}
})
.await;
result.expect("export ClickHouse query result to XLSX without per-page re-execution");
assert!(done_seen.load(Ordering::Relaxed));
let parsed = parse_xlsx_file(&file_path.to_string_lossy(), 3000).expect("parse exported XLSX");
let _ = std::fs::remove_dir_all(&dir);
assert_eq!(parsed.columns, vec!["id", "label"]);
assert_eq!(parsed.total_rows, 2050);
assert_eq!(parsed.rows.len(), 2050);
let exported_ids = parsed.rows.iter().map(|row| json_cell_i64(&row[0])).collect::<BTreeSet<_>>();
let expected_ids = (1..=2050).collect::<BTreeSet<_>>();
assert_eq!(exported_ids, expected_ids);
}