fix(clickhouse): stream query result exports
This commit is contained in:
1 parent
e87e31752a
commit
315c6e235b
3 files changed
+533
-2
No files matched your search
@@ -1,10 +1,12 @@
|
||||
use futures::StreamExt;
|
||||
use reqwest::{Certificate, Client as HttpClient};
|
||||
use serde::Deserialize;
|
||||
use std::fs;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::{http_client_builder, with_connection_timeout};
|
||||
use crate::query::MAX_ROWS;
|
||||
use crate::query::{canceled_error, MAX_ROWS};
|
||||
use crate::sql::starts_with_executable_sql_keyword;
|
||||
use crate::types::{ColumnInfo, DatabaseInfo, IndexInfo, ObjectStatistics, QueryResult, TableInfo};
|
||||
|
||||
@@ -100,8 +102,39 @@ enum QueryResultLimit {
|
||||
Limited(usize),
|
||||
}
|
||||
|
||||
enum QueryResultFormat {
|
||||
JsonCompact,
|
||||
JsonCompactEachRowWithNamesAndTypes,
|
||||
}
|
||||
|
||||
const CLICKHOUSE_STREAM_ROW_LIMIT_REACHED: &str = "__DBX_CLICKHOUSE_STREAM_ROW_LIMIT_REACHED__";
|
||||
|
||||
impl QueryResultFormat {
|
||||
fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
QueryResultFormat::JsonCompact => "JSONCompact",
|
||||
QueryResultFormat::JsonCompactEachRowWithNamesAndTypes => "JSONCompactEachRowWithNamesAndTypes",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum ClickHouseQueryStreamItem {
|
||||
Columns { columns: Vec<String>, column_types: Vec<String> },
|
||||
Row(Vec<serde_json::Value>),
|
||||
}
|
||||
|
||||
fn build_query_url(base_url: &str, database: Option<&str>, limit: QueryResultLimit) -> String {
|
||||
let mut url = format!("{}/?default_format=JSONCompact", base_url);
|
||||
build_query_url_with_format(base_url, database, limit, QueryResultFormat::JsonCompact)
|
||||
}
|
||||
|
||||
fn build_query_url_with_format(
|
||||
base_url: &str,
|
||||
database: Option<&str>,
|
||||
limit: QueryResultLimit,
|
||||
format: QueryResultFormat,
|
||||
) -> String {
|
||||
let mut url = format!("{}/?default_format={}", base_url, format.as_str());
|
||||
if let Some(db) = database {
|
||||
url.push_str(&format!("&database={db}"));
|
||||
}
|
||||
@@ -111,6 +144,94 @@ fn build_query_url(base_url: &str, database: Option<&str>, limit: QueryResultLim
|
||||
url
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ClickHouseStreamParser {
|
||||
line_index: usize,
|
||||
columns: Vec<String>,
|
||||
}
|
||||
|
||||
impl ClickHouseStreamParser {
|
||||
fn parse_line(&mut self, line: &str) -> Result<Option<ClickHouseQueryStreamItem>, String> {
|
||||
let line = line.trim_end_matches('\r');
|
||||
if line.trim().is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
match self.line_index {
|
||||
0 => {
|
||||
self.columns = parse_stream_string_array(line, "column names")?;
|
||||
self.line_index = 1;
|
||||
Ok(None)
|
||||
}
|
||||
1 => {
|
||||
let column_types = parse_stream_string_array(line, "column types")?;
|
||||
self.line_index = 2;
|
||||
Ok(Some(ClickHouseQueryStreamItem::Columns { columns: self.columns.clone(), column_types }))
|
||||
}
|
||||
_ => {
|
||||
let row = parse_stream_value_array(line, "row")?;
|
||||
self.line_index += 1;
|
||||
Ok(Some(ClickHouseQueryStreamItem::Row(row)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(&self) -> Result<(), String> {
|
||||
if self.line_index == 1 {
|
||||
return Err("ClickHouse stream ended before column types were received".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_stream_string_array(line: &str, label: &str) -> Result<Vec<String>, String> {
|
||||
parse_stream_value_array(line, label).map(|values| {
|
||||
values
|
||||
.into_iter()
|
||||
.map(|value| match value {
|
||||
serde_json::Value::String(value) => value,
|
||||
other => other.to_string(),
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_stream_value_array(line: &str, label: &str) -> Result<Vec<serde_json::Value>, String> {
|
||||
serde_json::from_str(line).map_err(|e| format!("ClickHouse stream parse error in {label}: {e}"))
|
||||
}
|
||||
|
||||
fn process_stream_buffer(
|
||||
buffer: &mut Vec<u8>,
|
||||
parser: &mut ClickHouseStreamParser,
|
||||
on_item: &mut impl FnMut(ClickHouseQueryStreamItem) -> Result<(), String>,
|
||||
) -> Result<(), String> {
|
||||
while let Some(newline) = buffer.iter().position(|byte| *byte == b'\n') {
|
||||
let line_bytes: Vec<u8> = buffer.drain(..=newline).collect();
|
||||
let line_bytes = &line_bytes[..line_bytes.len().saturating_sub(1)];
|
||||
let line =
|
||||
std::str::from_utf8(line_bytes).map_err(|e| format!("ClickHouse stream contains invalid UTF-8: {e}"))?;
|
||||
if let Some(item) = parser.parse_line(line)? {
|
||||
on_item(item)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn process_stream_remainder(
|
||||
buffer: &mut Vec<u8>,
|
||||
parser: &mut ClickHouseStreamParser,
|
||||
on_item: &mut impl FnMut(ClickHouseQueryStreamItem) -> Result<(), String>,
|
||||
) -> Result<(), String> {
|
||||
if !buffer.is_empty() {
|
||||
let line = std::str::from_utf8(buffer).map_err(|e| format!("ClickHouse stream contains invalid UTF-8: {e}"))?;
|
||||
if let Some(item) = parser.parse_line(line)? {
|
||||
on_item(item)?;
|
||||
}
|
||||
buffer.clear();
|
||||
}
|
||||
parser.finish()
|
||||
}
|
||||
|
||||
fn build_request(client: &ChClient, req: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
|
||||
match (&client.username, &client.password) {
|
||||
(Some(u), Some(p)) if !u.is_empty() => req.basic_auth(u, Some(p)),
|
||||
@@ -142,6 +263,75 @@ async fn ch_query_with_limit(
|
||||
resp.json::<ChJsonResult>().await.map_err(|e| format!("ClickHouse parse error: {e}"))
|
||||
}
|
||||
|
||||
pub async fn stream_query_with_max_rows(
|
||||
client: &ChClient,
|
||||
database: &str,
|
||||
sql: &str,
|
||||
max_rows: Option<usize>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
mut on_item: impl FnMut(ClickHouseQueryStreamItem) -> Result<(), String>,
|
||||
) -> Result<(), String> {
|
||||
let max_rows = max_rows.map(|max_rows| max_rows.max(1));
|
||||
let limit = max_rows.map(QueryResultLimit::Limited).unwrap_or(QueryResultLimit::Unlimited);
|
||||
let url = build_query_url_with_format(
|
||||
&client.base_url,
|
||||
Some(database),
|
||||
limit,
|
||||
QueryResultFormat::JsonCompactEachRowWithNamesAndTypes,
|
||||
);
|
||||
log::info!("[clickhouse] stream query url={url} user={:?} has_pass={}", client.username, client.password.is_some());
|
||||
let req = build_request(client, client.http.post(&url).body(sql.to_string()));
|
||||
let resp = req.send().await.map_err(|e| format!("ClickHouse request failed: {e}"))?;
|
||||
log::info!("[clickhouse] stream response status={}", resp.status());
|
||||
if !resp.status().is_success() {
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
log::error!("[clickhouse] stream error body: {body}");
|
||||
return Err(format!("ClickHouse error: {body}"));
|
||||
}
|
||||
|
||||
let mut stream = resp.bytes_stream();
|
||||
let mut buffer = Vec::new();
|
||||
let mut parser = ClickHouseStreamParser::default();
|
||||
let mut rows_seen = 0usize;
|
||||
let mut emit_item = |item: ClickHouseQueryStreamItem| -> Result<(), String> {
|
||||
if matches!(item, ClickHouseQueryStreamItem::Row(_)) {
|
||||
rows_seen = rows_seen.saturating_add(1);
|
||||
}
|
||||
on_item(item)?;
|
||||
if max_rows.is_some_and(|max_rows| rows_seen >= max_rows) {
|
||||
return Err(CLICKHOUSE_STREAM_ROW_LIMIT_REACHED.to_string());
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
|
||||
loop {
|
||||
let chunk = match cancel_token.as_ref() {
|
||||
Some(token) => {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => return Err(canceled_error()),
|
||||
chunk = stream.next() => chunk,
|
||||
}
|
||||
}
|
||||
None => stream.next().await,
|
||||
};
|
||||
let Some(chunk) = chunk else {
|
||||
break;
|
||||
};
|
||||
let chunk = chunk.map_err(|e| format!("ClickHouse stream read failed: {e}"))?;
|
||||
buffer.extend_from_slice(&chunk);
|
||||
match process_stream_buffer(&mut buffer, &mut parser, &mut emit_item) {
|
||||
Err(error) if error == CLICKHOUSE_STREAM_ROW_LIMIT_REACHED => return Ok(()),
|
||||
result => result?,
|
||||
}
|
||||
}
|
||||
|
||||
match process_stream_remainder(&mut buffer, &mut parser, &mut emit_item) {
|
||||
Err(error) if error == CLICKHOUSE_STREAM_ROW_LIMIT_REACHED => Ok(()),
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
|
||||
fn query_result_row_limit(max_rows: Option<usize>) -> usize {
|
||||
max_rows.unwrap_or(MAX_ROWS).max(1)
|
||||
}
|
||||
@@ -449,6 +639,52 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stream_query_url_uses_line_delimited_names_and_types_format() {
|
||||
let url = build_query_url_with_format(
|
||||
"http://localhost:8123",
|
||||
Some("analytics"),
|
||||
QueryResultLimit::Limited(500),
|
||||
QueryResultFormat::JsonCompactEachRowWithNamesAndTypes,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
url,
|
||||
"http://localhost:8123/?default_format=JSONCompactEachRowWithNamesAndTypes&database=analytics&max_result_rows=500&result_overflow_mode=break"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stream_parser_reads_columns_types_and_rows_across_chunks() {
|
||||
let mut buffer = b"[\"id\",\"name\"]\n[\"UInt64\",\"String\"]\n[1,\"Ada\"]\n[2,\"Linus\"".to_vec();
|
||||
let mut parser = ClickHouseStreamParser::default();
|
||||
let mut items = Vec::new();
|
||||
|
||||
process_stream_buffer(&mut buffer, &mut parser, &mut |item| {
|
||||
items.push(item);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
buffer.extend_from_slice(b"]\n");
|
||||
process_stream_remainder(&mut buffer, &mut parser, &mut |item| {
|
||||
items.push(item);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
items,
|
||||
vec![
|
||||
ClickHouseQueryStreamItem::Columns {
|
||||
columns: vec!["id".to_string(), "name".to_string()],
|
||||
column_types: vec!["UInt64".to_string(), "String".to_string()],
|
||||
},
|
||||
ClickHouseQueryStreamItem::Row(vec![serde_json::json!(1), serde_json::json!("Ada")]),
|
||||
ClickHouseQueryStreamItem::Row(vec![serde_json::json!(2), serde_json::json!("Linus")]),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limited_query_result_truncates_extra_probe_row() {
|
||||
let result = ChJsonResult {
|
||||
|
||||
@@ -333,6 +333,12 @@ async fn export_query_result_core_inner(
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// ClickHouse HTTP pagination is unsafe for unsorted result sets; stream one
|
||||
// response so large exports preserve the server's single execution order.
|
||||
if try_export_clickhouse_query_result_stream(state, request, &format, cancel_token.clone(), on_progress).await? {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut csv_file = if format == "csv" {
|
||||
Some(BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?))
|
||||
} else {
|
||||
@@ -950,6 +956,174 @@ async fn try_export_mysql_query_result_stream(
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn try_export_clickhouse_query_result_stream(
|
||||
state: &AppState,
|
||||
request: &QueryResultExportRequest,
|
||||
format: &str,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
on_progress: &impl Fn(TableExportProgress),
|
||||
) -> Result<bool, String> {
|
||||
if request.database_type != DatabaseType::ClickHouse
|
||||
|| request.use_agent_cursor
|
||||
|| !crate::sql::starts_with_executable_sql_keyword(
|
||||
&request.sql,
|
||||
&["SELECT", "SHOW", "DESCRIBE", "EXPLAIN", "WITH"],
|
||||
)
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let database = request.database.trim();
|
||||
let pool_key = if database.is_empty() {
|
||||
state.get_or_create_pool_for_session(&request.connection_id, None, request.client_session_id.as_deref()).await?
|
||||
} else {
|
||||
state
|
||||
.get_or_create_pool_for_session(
|
||||
&request.connection_id,
|
||||
Some(database),
|
||||
request.client_session_id.as_deref(),
|
||||
)
|
||||
.await?
|
||||
};
|
||||
let connections = state.connections.read().await;
|
||||
let Some(client) = connections.get(&pool_key).and_then(|pool| match pool {
|
||||
PoolKind::ClickHouse(client) => Some(client.clone()),
|
||||
_ => None,
|
||||
}) else {
|
||||
return Ok(false);
|
||||
};
|
||||
drop(connections);
|
||||
|
||||
if let Some(execution_id) = request.execution_id.as_deref() {
|
||||
state.running_queries.set_pool_key(execution_id, pool_key.clone());
|
||||
}
|
||||
state.touch_pool_activity(&pool_key).await;
|
||||
let _activity_touch = state.pool_activity_touch(&pool_key);
|
||||
|
||||
let xlsx_hard_limit_active = xlsx_hard_limit_active(format, request);
|
||||
let row_limit = effective_row_limit(format, request);
|
||||
let stream_row_limit =
|
||||
if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit };
|
||||
let progress_row_interval = request.page_size.max(1) as u64;
|
||||
let mut columns: Vec<String> = Vec::new();
|
||||
let mut rows_exported = 0_u64;
|
||||
let mut last_progress_rows = 0_u64;
|
||||
let mut last_progress_at = Instant::now();
|
||||
let mut csv_file = if format == "csv" {
|
||||
let mut file =
|
||||
BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?);
|
||||
file.write_all(b"\xEF\xBB\xBF").map_err(|e| format!("Failed to write BOM: {e}"))?;
|
||||
Some(file)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut xlsx = None;
|
||||
let clickhouse_database = if database.is_empty() { "default" } else { database };
|
||||
|
||||
let stream_future = crate::db::clickhouse_driver::stream_query_with_max_rows(
|
||||
&client,
|
||||
clickhouse_database,
|
||||
&request.sql,
|
||||
stream_row_limit,
|
||||
cancel_token.clone(),
|
||||
|item| {
|
||||
match item {
|
||||
crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Columns {
|
||||
columns: stream_columns, ..
|
||||
} => {
|
||||
columns = stream_columns;
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
let csv = format_query_result_csv(&columns, &[]);
|
||||
let header = csv.strip_suffix('\n').unwrap_or(&csv);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||
} else {
|
||||
let xlsx_file =
|
||||
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||
xlsx =
|
||||
Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
|
||||
}
|
||||
}
|
||||
crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Row(row) => {
|
||||
if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS {
|
||||
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
|
||||
}
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
let rows_csv = format_query_result_csv_rows(std::slice::from_ref(&row));
|
||||
write!(file, "\n{rows_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?;
|
||||
} else if let Some(writer) = xlsx.as_mut() {
|
||||
writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
} else {
|
||||
let xlsx_file =
|
||||
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||
xlsx =
|
||||
Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
|
||||
if let Some(writer) = xlsx.as_mut() {
|
||||
writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
}
|
||||
}
|
||||
rows_exported += 1;
|
||||
let now = Instant::now();
|
||||
if should_emit_stream_progress(
|
||||
rows_exported,
|
||||
last_progress_rows,
|
||||
progress_row_interval,
|
||||
now.duration_since(last_progress_at),
|
||||
) {
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Running, None));
|
||||
last_progress_rows = rows_exported;
|
||||
last_progress_at = now;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
);
|
||||
let stream_result = match query_export_timeout(request.timeout_secs) {
|
||||
Some(timeout) => match tokio::time::timeout(timeout, stream_future).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(format!("Query timed out after {} seconds", timeout.as_secs())),
|
||||
},
|
||||
None => stream_future.await,
|
||||
};
|
||||
|
||||
if let Err(error) = stream_result {
|
||||
if error == QUERY_CANCELED
|
||||
|| cancel_token.as_ref().is_some_and(|token| token.is_cancelled())
|
||||
|| is_export_cancelled(&request.export_id).await
|
||||
{
|
||||
on_progress(progress(
|
||||
request,
|
||||
rows_exported,
|
||||
ExportStatus::Cancelled,
|
||||
Some("Export cancelled".to_string()),
|
||||
));
|
||||
return Ok(true);
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
if rows_exported != last_progress_rows {
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Running, None));
|
||||
}
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Writing, None));
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
file.flush().map_err(|e| format!("Failed to flush CSV file: {e}"))?;
|
||||
}
|
||||
if let Some(writer) = xlsx {
|
||||
let mut buf =
|
||||
finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?;
|
||||
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
|
||||
} else if format == "xlsx" {
|
||||
let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||
let writer = start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?;
|
||||
let mut buf =
|
||||
finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?;
|
||||
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
|
||||
}
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Done, None));
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn try_export_sqlserver_query_result_stream(
|
||||
state: &AppState,
|
||||
request: &QueryResultExportRequest,
|
||||
|
||||
Reference in new issue
Block a user