fix(clickhouse): stream query result exports
This commit is contained in:
1 parent
e87e31752a
commit
315c6e235b
3 files changed
+533
-2
No files matched your search
@@ -1,10 +1,12 @@
|
|||||||
|
use futures::StreamExt;
|
||||||
use reqwest::{Certificate, Client as HttpClient};
|
use reqwest::{Certificate, Client as HttpClient};
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
use tokio_util::sync::CancellationToken;
|
||||||
|
|
||||||
use super::{http_client_builder, with_connection_timeout};
|
use super::{http_client_builder, with_connection_timeout};
|
||||||
use crate::query::MAX_ROWS;
|
use crate::query::{canceled_error, MAX_ROWS};
|
||||||
use crate::sql::starts_with_executable_sql_keyword;
|
use crate::sql::starts_with_executable_sql_keyword;
|
||||||
use crate::types::{ColumnInfo, DatabaseInfo, IndexInfo, ObjectStatistics, QueryResult, TableInfo};
|
use crate::types::{ColumnInfo, DatabaseInfo, IndexInfo, ObjectStatistics, QueryResult, TableInfo};
|
||||||
|
|
||||||
@@ -100,8 +102,39 @@ enum QueryResultLimit {
|
|||||||
Limited(usize),
|
Limited(usize),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
enum QueryResultFormat {
|
||||||
|
JsonCompact,
|
||||||
|
JsonCompactEachRowWithNamesAndTypes,
|
||||||
|
}
|
||||||
|
|
||||||
|
const CLICKHOUSE_STREAM_ROW_LIMIT_REACHED: &str = "__DBX_CLICKHOUSE_STREAM_ROW_LIMIT_REACHED__";
|
||||||
|
|
||||||
|
impl QueryResultFormat {
|
||||||
|
fn as_str(&self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
QueryResultFormat::JsonCompact => "JSONCompact",
|
||||||
|
QueryResultFormat::JsonCompactEachRowWithNamesAndTypes => "JSONCompactEachRowWithNamesAndTypes",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
|
pub enum ClickHouseQueryStreamItem {
|
||||||
|
Columns { columns: Vec<String>, column_types: Vec<String> },
|
||||||
|
Row(Vec<serde_json::Value>),
|
||||||
|
}
|
||||||
|
|
||||||
fn build_query_url(base_url: &str, database: Option<&str>, limit: QueryResultLimit) -> String {
|
fn build_query_url(base_url: &str, database: Option<&str>, limit: QueryResultLimit) -> String {
|
||||||
let mut url = format!("{}/?default_format=JSONCompact", base_url);
|
build_query_url_with_format(base_url, database, limit, QueryResultFormat::JsonCompact)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_query_url_with_format(
|
||||||
|
base_url: &str,
|
||||||
|
database: Option<&str>,
|
||||||
|
limit: QueryResultLimit,
|
||||||
|
format: QueryResultFormat,
|
||||||
|
) -> String {
|
||||||
|
let mut url = format!("{}/?default_format={}", base_url, format.as_str());
|
||||||
if let Some(db) = database {
|
if let Some(db) = database {
|
||||||
url.push_str(&format!("&database={db}"));
|
url.push_str(&format!("&database={db}"));
|
||||||
}
|
}
|
||||||
@@ -111,6 +144,94 @@ fn build_query_url(base_url: &str, database: Option<&str>, limit: QueryResultLim
|
|||||||
url
|
url
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct ClickHouseStreamParser {
|
||||||
|
line_index: usize,
|
||||||
|
columns: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ClickHouseStreamParser {
|
||||||
|
fn parse_line(&mut self, line: &str) -> Result<Option<ClickHouseQueryStreamItem>, String> {
|
||||||
|
let line = line.trim_end_matches('\r');
|
||||||
|
if line.trim().is_empty() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
match self.line_index {
|
||||||
|
0 => {
|
||||||
|
self.columns = parse_stream_string_array(line, "column names")?;
|
||||||
|
self.line_index = 1;
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
1 => {
|
||||||
|
let column_types = parse_stream_string_array(line, "column types")?;
|
||||||
|
self.line_index = 2;
|
||||||
|
Ok(Some(ClickHouseQueryStreamItem::Columns { columns: self.columns.clone(), column_types }))
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
let row = parse_stream_value_array(line, "row")?;
|
||||||
|
self.line_index += 1;
|
||||||
|
Ok(Some(ClickHouseQueryStreamItem::Row(row)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn finish(&self) -> Result<(), String> {
|
||||||
|
if self.line_index == 1 {
|
||||||
|
return Err("ClickHouse stream ended before column types were received".to_string());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_stream_string_array(line: &str, label: &str) -> Result<Vec<String>, String> {
|
||||||
|
parse_stream_value_array(line, label).map(|values| {
|
||||||
|
values
|
||||||
|
.into_iter()
|
||||||
|
.map(|value| match value {
|
||||||
|
serde_json::Value::String(value) => value,
|
||||||
|
other => other.to_string(),
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_stream_value_array(line: &str, label: &str) -> Result<Vec<serde_json::Value>, String> {
|
||||||
|
serde_json::from_str(line).map_err(|e| format!("ClickHouse stream parse error in {label}: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn process_stream_buffer(
|
||||||
|
buffer: &mut Vec<u8>,
|
||||||
|
parser: &mut ClickHouseStreamParser,
|
||||||
|
on_item: &mut impl FnMut(ClickHouseQueryStreamItem) -> Result<(), String>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
while let Some(newline) = buffer.iter().position(|byte| *byte == b'\n') {
|
||||||
|
let line_bytes: Vec<u8> = buffer.drain(..=newline).collect();
|
||||||
|
let line_bytes = &line_bytes[..line_bytes.len().saturating_sub(1)];
|
||||||
|
let line =
|
||||||
|
std::str::from_utf8(line_bytes).map_err(|e| format!("ClickHouse stream contains invalid UTF-8: {e}"))?;
|
||||||
|
if let Some(item) = parser.parse_line(line)? {
|
||||||
|
on_item(item)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn process_stream_remainder(
|
||||||
|
buffer: &mut Vec<u8>,
|
||||||
|
parser: &mut ClickHouseStreamParser,
|
||||||
|
on_item: &mut impl FnMut(ClickHouseQueryStreamItem) -> Result<(), String>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
if !buffer.is_empty() {
|
||||||
|
let line = std::str::from_utf8(buffer).map_err(|e| format!("ClickHouse stream contains invalid UTF-8: {e}"))?;
|
||||||
|
if let Some(item) = parser.parse_line(line)? {
|
||||||
|
on_item(item)?;
|
||||||
|
}
|
||||||
|
buffer.clear();
|
||||||
|
}
|
||||||
|
parser.finish()
|
||||||
|
}
|
||||||
|
|
||||||
fn build_request(client: &ChClient, req: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
|
fn build_request(client: &ChClient, req: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
|
||||||
match (&client.username, &client.password) {
|
match (&client.username, &client.password) {
|
||||||
(Some(u), Some(p)) if !u.is_empty() => req.basic_auth(u, Some(p)),
|
(Some(u), Some(p)) if !u.is_empty() => req.basic_auth(u, Some(p)),
|
||||||
@@ -142,6 +263,75 @@ async fn ch_query_with_limit(
|
|||||||
resp.json::<ChJsonResult>().await.map_err(|e| format!("ClickHouse parse error: {e}"))
|
resp.json::<ChJsonResult>().await.map_err(|e| format!("ClickHouse parse error: {e}"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn stream_query_with_max_rows(
|
||||||
|
client: &ChClient,
|
||||||
|
database: &str,
|
||||||
|
sql: &str,
|
||||||
|
max_rows: Option<usize>,
|
||||||
|
cancel_token: Option<CancellationToken>,
|
||||||
|
mut on_item: impl FnMut(ClickHouseQueryStreamItem) -> Result<(), String>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let max_rows = max_rows.map(|max_rows| max_rows.max(1));
|
||||||
|
let limit = max_rows.map(QueryResultLimit::Limited).unwrap_or(QueryResultLimit::Unlimited);
|
||||||
|
let url = build_query_url_with_format(
|
||||||
|
&client.base_url,
|
||||||
|
Some(database),
|
||||||
|
limit,
|
||||||
|
QueryResultFormat::JsonCompactEachRowWithNamesAndTypes,
|
||||||
|
);
|
||||||
|
log::info!("[clickhouse] stream query url={url} user={:?} has_pass={}", client.username, client.password.is_some());
|
||||||
|
let req = build_request(client, client.http.post(&url).body(sql.to_string()));
|
||||||
|
let resp = req.send().await.map_err(|e| format!("ClickHouse request failed: {e}"))?;
|
||||||
|
log::info!("[clickhouse] stream response status={}", resp.status());
|
||||||
|
if !resp.status().is_success() {
|
||||||
|
let body = resp.text().await.unwrap_or_default();
|
||||||
|
log::error!("[clickhouse] stream error body: {body}");
|
||||||
|
return Err(format!("ClickHouse error: {body}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut stream = resp.bytes_stream();
|
||||||
|
let mut buffer = Vec::new();
|
||||||
|
let mut parser = ClickHouseStreamParser::default();
|
||||||
|
let mut rows_seen = 0usize;
|
||||||
|
let mut emit_item = |item: ClickHouseQueryStreamItem| -> Result<(), String> {
|
||||||
|
if matches!(item, ClickHouseQueryStreamItem::Row(_)) {
|
||||||
|
rows_seen = rows_seen.saturating_add(1);
|
||||||
|
}
|
||||||
|
on_item(item)?;
|
||||||
|
if max_rows.is_some_and(|max_rows| rows_seen >= max_rows) {
|
||||||
|
return Err(CLICKHOUSE_STREAM_ROW_LIMIT_REACHED.to_string());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
};
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let chunk = match cancel_token.as_ref() {
|
||||||
|
Some(token) => {
|
||||||
|
tokio::select! {
|
||||||
|
biased;
|
||||||
|
_ = token.cancelled() => return Err(canceled_error()),
|
||||||
|
chunk = stream.next() => chunk,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => stream.next().await,
|
||||||
|
};
|
||||||
|
let Some(chunk) = chunk else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
let chunk = chunk.map_err(|e| format!("ClickHouse stream read failed: {e}"))?;
|
||||||
|
buffer.extend_from_slice(&chunk);
|
||||||
|
match process_stream_buffer(&mut buffer, &mut parser, &mut emit_item) {
|
||||||
|
Err(error) if error == CLICKHOUSE_STREAM_ROW_LIMIT_REACHED => return Ok(()),
|
||||||
|
result => result?,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
match process_stream_remainder(&mut buffer, &mut parser, &mut emit_item) {
|
||||||
|
Err(error) if error == CLICKHOUSE_STREAM_ROW_LIMIT_REACHED => Ok(()),
|
||||||
|
result => result,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn query_result_row_limit(max_rows: Option<usize>) -> usize {
|
fn query_result_row_limit(max_rows: Option<usize>) -> usize {
|
||||||
max_rows.unwrap_or(MAX_ROWS).max(1)
|
max_rows.unwrap_or(MAX_ROWS).max(1)
|
||||||
}
|
}
|
||||||
@@ -449,6 +639,52 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stream_query_url_uses_line_delimited_names_and_types_format() {
|
||||||
|
let url = build_query_url_with_format(
|
||||||
|
"http://localhost:8123",
|
||||||
|
Some("analytics"),
|
||||||
|
QueryResultLimit::Limited(500),
|
||||||
|
QueryResultFormat::JsonCompactEachRowWithNamesAndTypes,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
url,
|
||||||
|
"http://localhost:8123/?default_format=JSONCompactEachRowWithNamesAndTypes&database=analytics&max_result_rows=500&result_overflow_mode=break"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stream_parser_reads_columns_types_and_rows_across_chunks() {
|
||||||
|
let mut buffer = b"[\"id\",\"name\"]\n[\"UInt64\",\"String\"]\n[1,\"Ada\"]\n[2,\"Linus\"".to_vec();
|
||||||
|
let mut parser = ClickHouseStreamParser::default();
|
||||||
|
let mut items = Vec::new();
|
||||||
|
|
||||||
|
process_stream_buffer(&mut buffer, &mut parser, &mut |item| {
|
||||||
|
items.push(item);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
buffer.extend_from_slice(b"]\n");
|
||||||
|
process_stream_remainder(&mut buffer, &mut parser, &mut |item| {
|
||||||
|
items.push(item);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
items,
|
||||||
|
vec![
|
||||||
|
ClickHouseQueryStreamItem::Columns {
|
||||||
|
columns: vec!["id".to_string(), "name".to_string()],
|
||||||
|
column_types: vec!["UInt64".to_string(), "String".to_string()],
|
||||||
|
},
|
||||||
|
ClickHouseQueryStreamItem::Row(vec![serde_json::json!(1), serde_json::json!("Ada")]),
|
||||||
|
ClickHouseQueryStreamItem::Row(vec![serde_json::json!(2), serde_json::json!("Linus")]),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn limited_query_result_truncates_extra_probe_row() {
|
fn limited_query_result_truncates_extra_probe_row() {
|
||||||
let result = ChJsonResult {
|
let result = ChJsonResult {
|
||||||
|
|||||||
@@ -333,6 +333,12 @@ async fn export_query_result_core_inner(
|
|||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ClickHouse HTTP pagination is unsafe for unsorted result sets; stream one
|
||||||
|
// response so large exports preserve the server's single execution order.
|
||||||
|
if try_export_clickhouse_query_result_stream(state, request, &format, cancel_token.clone(), on_progress).await? {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
let mut csv_file = if format == "csv" {
|
let mut csv_file = if format == "csv" {
|
||||||
Some(BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?))
|
Some(BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?))
|
||||||
} else {
|
} else {
|
||||||
@@ -950,6 +956,174 @@ async fn try_export_mysql_query_result_stream(
|
|||||||
Ok(true)
|
Ok(true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn try_export_clickhouse_query_result_stream(
|
||||||
|
state: &AppState,
|
||||||
|
request: &QueryResultExportRequest,
|
||||||
|
format: &str,
|
||||||
|
cancel_token: Option<CancellationToken>,
|
||||||
|
on_progress: &impl Fn(TableExportProgress),
|
||||||
|
) -> Result<bool, String> {
|
||||||
|
if request.database_type != DatabaseType::ClickHouse
|
||||||
|
|| request.use_agent_cursor
|
||||||
|
|| !crate::sql::starts_with_executable_sql_keyword(
|
||||||
|
&request.sql,
|
||||||
|
&["SELECT", "SHOW", "DESCRIBE", "EXPLAIN", "WITH"],
|
||||||
|
)
|
||||||
|
{
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
let database = request.database.trim();
|
||||||
|
let pool_key = if database.is_empty() {
|
||||||
|
state.get_or_create_pool_for_session(&request.connection_id, None, request.client_session_id.as_deref()).await?
|
||||||
|
} else {
|
||||||
|
state
|
||||||
|
.get_or_create_pool_for_session(
|
||||||
|
&request.connection_id,
|
||||||
|
Some(database),
|
||||||
|
request.client_session_id.as_deref(),
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
};
|
||||||
|
let connections = state.connections.read().await;
|
||||||
|
let Some(client) = connections.get(&pool_key).and_then(|pool| match pool {
|
||||||
|
PoolKind::ClickHouse(client) => Some(client.clone()),
|
||||||
|
_ => None,
|
||||||
|
}) else {
|
||||||
|
return Ok(false);
|
||||||
|
};
|
||||||
|
drop(connections);
|
||||||
|
|
||||||
|
if let Some(execution_id) = request.execution_id.as_deref() {
|
||||||
|
state.running_queries.set_pool_key(execution_id, pool_key.clone());
|
||||||
|
}
|
||||||
|
state.touch_pool_activity(&pool_key).await;
|
||||||
|
let _activity_touch = state.pool_activity_touch(&pool_key);
|
||||||
|
|
||||||
|
let xlsx_hard_limit_active = xlsx_hard_limit_active(format, request);
|
||||||
|
let row_limit = effective_row_limit(format, request);
|
||||||
|
let stream_row_limit =
|
||||||
|
if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit };
|
||||||
|
let progress_row_interval = request.page_size.max(1) as u64;
|
||||||
|
let mut columns: Vec<String> = Vec::new();
|
||||||
|
let mut rows_exported = 0_u64;
|
||||||
|
let mut last_progress_rows = 0_u64;
|
||||||
|
let mut last_progress_at = Instant::now();
|
||||||
|
let mut csv_file = if format == "csv" {
|
||||||
|
let mut file =
|
||||||
|
BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?);
|
||||||
|
file.write_all(b"\xEF\xBB\xBF").map_err(|e| format!("Failed to write BOM: {e}"))?;
|
||||||
|
Some(file)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let mut xlsx = None;
|
||||||
|
let clickhouse_database = if database.is_empty() { "default" } else { database };
|
||||||
|
|
||||||
|
let stream_future = crate::db::clickhouse_driver::stream_query_with_max_rows(
|
||||||
|
&client,
|
||||||
|
clickhouse_database,
|
||||||
|
&request.sql,
|
||||||
|
stream_row_limit,
|
||||||
|
cancel_token.clone(),
|
||||||
|
|item| {
|
||||||
|
match item {
|
||||||
|
crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Columns {
|
||||||
|
columns: stream_columns, ..
|
||||||
|
} => {
|
||||||
|
columns = stream_columns;
|
||||||
|
if let Some(file) = csv_file.as_mut() {
|
||||||
|
let csv = format_query_result_csv(&columns, &[]);
|
||||||
|
let header = csv.strip_suffix('\n').unwrap_or(&csv);
|
||||||
|
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||||
|
} else {
|
||||||
|
let xlsx_file =
|
||||||
|
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||||
|
xlsx =
|
||||||
|
Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Row(row) => {
|
||||||
|
if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS {
|
||||||
|
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
|
||||||
|
}
|
||||||
|
if let Some(file) = csv_file.as_mut() {
|
||||||
|
let rows_csv = format_query_result_csv_rows(std::slice::from_ref(&row));
|
||||||
|
write!(file, "\n{rows_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?;
|
||||||
|
} else if let Some(writer) = xlsx.as_mut() {
|
||||||
|
writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||||
|
} else {
|
||||||
|
let xlsx_file =
|
||||||
|
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||||
|
xlsx =
|
||||||
|
Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
|
||||||
|
if let Some(writer) = xlsx.as_mut() {
|
||||||
|
writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rows_exported += 1;
|
||||||
|
let now = Instant::now();
|
||||||
|
if should_emit_stream_progress(
|
||||||
|
rows_exported,
|
||||||
|
last_progress_rows,
|
||||||
|
progress_row_interval,
|
||||||
|
now.duration_since(last_progress_at),
|
||||||
|
) {
|
||||||
|
on_progress(progress(request, rows_exported, ExportStatus::Running, None));
|
||||||
|
last_progress_rows = rows_exported;
|
||||||
|
last_progress_at = now;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let stream_result = match query_export_timeout(request.timeout_secs) {
|
||||||
|
Some(timeout) => match tokio::time::timeout(timeout, stream_future).await {
|
||||||
|
Ok(result) => result,
|
||||||
|
Err(_) => Err(format!("Query timed out after {} seconds", timeout.as_secs())),
|
||||||
|
},
|
||||||
|
None => stream_future.await,
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Err(error) = stream_result {
|
||||||
|
if error == QUERY_CANCELED
|
||||||
|
|| cancel_token.as_ref().is_some_and(|token| token.is_cancelled())
|
||||||
|
|| is_export_cancelled(&request.export_id).await
|
||||||
|
{
|
||||||
|
on_progress(progress(
|
||||||
|
request,
|
||||||
|
rows_exported,
|
||||||
|
ExportStatus::Cancelled,
|
||||||
|
Some("Export cancelled".to_string()),
|
||||||
|
));
|
||||||
|
return Ok(true);
|
||||||
|
}
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
if rows_exported != last_progress_rows {
|
||||||
|
on_progress(progress(request, rows_exported, ExportStatus::Running, None));
|
||||||
|
}
|
||||||
|
on_progress(progress(request, rows_exported, ExportStatus::Writing, None));
|
||||||
|
if let Some(file) = csv_file.as_mut() {
|
||||||
|
file.flush().map_err(|e| format!("Failed to flush CSV file: {e}"))?;
|
||||||
|
}
|
||||||
|
if let Some(writer) = xlsx {
|
||||||
|
let mut buf =
|
||||||
|
finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?;
|
||||||
|
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
|
||||||
|
} else if format == "xlsx" {
|
||||||
|
let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||||
|
let writer = start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?;
|
||||||
|
let mut buf =
|
||||||
|
finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?;
|
||||||
|
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
|
||||||
|
}
|
||||||
|
on_progress(progress(request, rows_exported, ExportStatus::Done, None));
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
async fn try_export_sqlserver_query_result_stream(
|
async fn try_export_sqlserver_query_result_stream(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
request: &QueryResultExportRequest,
|
request: &QueryResultExportRequest,
|
||||||
|
|||||||
@@ -0,0 +1,121 @@
|
|||||||
|
use std::collections::BTreeSet;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
|
||||||
|
use dbx_core::connection::AppState;
|
||||||
|
use dbx_core::models::connection::{ConnectionConfig, DatabaseType};
|
||||||
|
use dbx_core::query_result_export::{export_query_result_core, ExportStatus, QueryResultExportRequest};
|
||||||
|
use dbx_core::storage::Storage;
|
||||||
|
use dbx_core::table_import::parse_xlsx_file;
|
||||||
|
|
||||||
|
fn live_clickhouse_config(
|
||||||
|
id: &str,
|
||||||
|
host: &str,
|
||||||
|
port: u16,
|
||||||
|
user: &str,
|
||||||
|
password: &str,
|
||||||
|
database: &str,
|
||||||
|
) -> ConnectionConfig {
|
||||||
|
serde_json::from_value(serde_json::json!({
|
||||||
|
"id": id,
|
||||||
|
"name": id,
|
||||||
|
"db_type": DatabaseType::ClickHouse,
|
||||||
|
"host": host,
|
||||||
|
"port": port,
|
||||||
|
"username": user,
|
||||||
|
"password": password,
|
||||||
|
"database": database,
|
||||||
|
"connect_timeout_secs": 10,
|
||||||
|
"query_timeout_secs": 30,
|
||||||
|
"idle_timeout_secs": 60,
|
||||||
|
"keepalive_interval_secs": 0
|
||||||
|
}))
|
||||||
|
.expect("live ClickHouse config should deserialize")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn json_cell_text(value: &serde_json::Value) -> String {
|
||||||
|
match value {
|
||||||
|
serde_json::Value::String(value) => value.clone(),
|
||||||
|
serde_json::Value::Number(value) => value.to_string(),
|
||||||
|
serde_json::Value::Bool(value) => value.to_string(),
|
||||||
|
serde_json::Value::Null => String::new(),
|
||||||
|
other => other.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn json_cell_i64(value: &serde_json::Value) -> i64 {
|
||||||
|
if let Some(value) = value.as_i64() {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
if let Some(value) = value.as_u64() {
|
||||||
|
return i64::try_from(value).expect("u64 id fits in i64");
|
||||||
|
}
|
||||||
|
let text = json_cell_text(value);
|
||||||
|
text.parse::<i64>().or_else(|_| text.parse::<f64>().map(|value| value as i64)).expect("numeric id")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore = "requires DBX_LIVE_CLICKHOUSE_* env vars pointing at a ClickHouse HTTP endpoint"]
|
||||||
|
async fn live_clickhouse_query_result_export_xlsx_streams_random_order_query_once() {
|
||||||
|
let host = std::env::var("DBX_LIVE_CLICKHOUSE_HOST").expect("DBX_LIVE_CLICKHOUSE_HOST");
|
||||||
|
let port = std::env::var("DBX_LIVE_CLICKHOUSE_PORT").expect("DBX_LIVE_CLICKHOUSE_PORT").parse::<u16>().unwrap();
|
||||||
|
let user = std::env::var("DBX_LIVE_CLICKHOUSE_USER").expect("DBX_LIVE_CLICKHOUSE_USER");
|
||||||
|
let password = std::env::var("DBX_LIVE_CLICKHOUSE_PASSWORD").expect("DBX_LIVE_CLICKHOUSE_PASSWORD");
|
||||||
|
let database = std::env::var("DBX_LIVE_CLICKHOUSE_DATABASE").expect("DBX_LIVE_CLICKHOUSE_DATABASE");
|
||||||
|
let suffix = uuid::Uuid::new_v4().simple().to_string();
|
||||||
|
let connection_id = format!("live-clickhouse-query-export-{suffix}");
|
||||||
|
let config = live_clickhouse_config(&connection_id, &host, port, &user, &password, &database);
|
||||||
|
let dir = std::env::temp_dir().join(format!("dbx-live-clickhouse-query-export-{suffix}"));
|
||||||
|
std::fs::create_dir_all(&dir).unwrap();
|
||||||
|
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||||
|
let state = AppState::new(storage);
|
||||||
|
state.configs.write().await.insert(config.id.clone(), config);
|
||||||
|
|
||||||
|
let file_path = dir.join("result.xlsx");
|
||||||
|
let sql = "
|
||||||
|
SELECT
|
||||||
|
number + 1 AS id,
|
||||||
|
concat('row-', toString(number + 1)) AS label
|
||||||
|
FROM numbers(2050)
|
||||||
|
ORDER BY rand()
|
||||||
|
"
|
||||||
|
.to_string();
|
||||||
|
let request = QueryResultExportRequest {
|
||||||
|
export_id: format!("live-clickhouse-query-export-{suffix}"),
|
||||||
|
connection_id: connection_id.clone(),
|
||||||
|
database: database.clone(),
|
||||||
|
schema: None,
|
||||||
|
sql: sql.clone(),
|
||||||
|
query_base_sql: sql,
|
||||||
|
database_type: DatabaseType::ClickHouse,
|
||||||
|
use_agent_cursor: false,
|
||||||
|
file_path: file_path.to_string_lossy().to_string(),
|
||||||
|
format: "xlsx".to_string(),
|
||||||
|
page_size: 100,
|
||||||
|
row_limit: None,
|
||||||
|
total_rows: Some(2050),
|
||||||
|
timeout_secs: Some(30),
|
||||||
|
keyset_optimization_enabled: false,
|
||||||
|
client_session_id: None,
|
||||||
|
execution_id: Some(format!("live-clickhouse-query-export-{suffix}")),
|
||||||
|
};
|
||||||
|
let done_seen = AtomicBool::new(false);
|
||||||
|
let result = export_query_result_core(&state, &request, None, |progress| {
|
||||||
|
if matches!(progress.status, ExportStatus::Done) {
|
||||||
|
done_seen.store(true, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
|
||||||
|
result.expect("export ClickHouse query result to XLSX without per-page re-execution");
|
||||||
|
assert!(done_seen.load(Ordering::Relaxed));
|
||||||
|
|
||||||
|
let parsed = parse_xlsx_file(&file_path.to_string_lossy(), 3000).expect("parse exported XLSX");
|
||||||
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
|
assert_eq!(parsed.columns, vec!["id", "label"]);
|
||||||
|
assert_eq!(parsed.total_rows, 2050);
|
||||||
|
assert_eq!(parsed.rows.len(), 2050);
|
||||||
|
|
||||||
|
let exported_ids = parsed.rows.iter().map(|row| json_cell_i64(&row[0])).collect::<BTreeSet<_>>();
|
||||||
|
let expected_ids = (1..=2050).collect::<BTreeSet<_>>();
|
||||||
|
assert_eq!(exported_ids, expected_ids);
|
||||||
|
}
|
||||||
Reference in new issue
Block a user