2 files changed
+251
-9
No files matched your search
@@ -119,6 +119,7 @@ struct ChColumn {
|
||||
_type: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum QueryResultLimit {
|
||||
Unlimited,
|
||||
Limited(usize),
|
||||
@@ -283,6 +284,10 @@ async fn ch_query_with_limit(
|
||||
database: Option<&str>,
|
||||
limit: QueryResultLimit,
|
||||
) -> Result<ChJsonResult, String> {
|
||||
let max_rows = match limit {
|
||||
QueryResultLimit::Unlimited => None,
|
||||
QueryResultLimit::Limited(max_rows) => Some(max_rows),
|
||||
};
|
||||
let url = build_query_url(&client.base_url, database, limit, client.extra_params.as_deref());
|
||||
log::info!("[clickhouse] query url={url} user={:?} has_pass={}", client.username, client.password.is_some());
|
||||
let req = build_request(client, client.http.post(&url).body(sql.to_string()));
|
||||
@@ -291,24 +296,68 @@ async fn ch_query_with_limit(
|
||||
if !resp.status().is_success() {
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
log::error!("[clickhouse] error body: {body}");
|
||||
return Err(format!("ClickHouse error: {body}"));
|
||||
let error = format!("ClickHouse error: {body}");
|
||||
if let Some(max_rows) = max_rows.filter(|_| is_readonly_result_limit_error(&error)) {
|
||||
log::warn!(
|
||||
"[clickhouse] server-side result limit rejected by readonly profile; retrying with client-side limit"
|
||||
);
|
||||
return ch_query_with_client_limit(client, sql, database, max_rows).await;
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
resp.json::<ChJsonResult>().await.map_err(|e| format!("ClickHouse parse error: {e}"))
|
||||
}
|
||||
|
||||
pub async fn stream_query_with_max_rows(
|
||||
fn is_readonly_result_limit_error(error: &str) -> bool {
|
||||
let readonly_error = error.contains("Code: 164") || error.contains("(READONLY)");
|
||||
readonly_error
|
||||
&& ["max_result_rows", "result_overflow_mode"]
|
||||
.iter()
|
||||
.any(|setting| error.contains(&format!("Cannot modify '{setting}' setting in readonly mode")))
|
||||
}
|
||||
|
||||
async fn ch_query_with_client_limit(
|
||||
client: &ChClient,
|
||||
database: &str,
|
||||
sql: &str,
|
||||
database: Option<&str>,
|
||||
max_rows: usize,
|
||||
) -> Result<ChJsonResult, String> {
|
||||
let mut meta = Vec::new();
|
||||
let mut data = Vec::new();
|
||||
let mut collect_item = |item| {
|
||||
match item {
|
||||
ClickHouseQueryStreamItem::Columns { columns, column_types } => {
|
||||
if columns.len() != column_types.len() {
|
||||
return Err("ClickHouse stream returned mismatched column names and types".to_string());
|
||||
}
|
||||
meta = columns.into_iter().zip(column_types).map(|(name, _type)| ChColumn { name, _type }).collect();
|
||||
}
|
||||
ClickHouseQueryStreamItem::Row(row) => data.push(row),
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
|
||||
stream_query_once(client, database, sql, Some(max_rows), None, false, &mut collect_item).await?;
|
||||
Ok(ChJsonResult { rows: data.len(), meta, data })
|
||||
}
|
||||
|
||||
async fn stream_query_once(
|
||||
client: &ChClient,
|
||||
database: Option<&str>,
|
||||
sql: &str,
|
||||
max_rows: Option<usize>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
mut on_item: impl FnMut(ClickHouseQueryStreamItem) -> Result<(), String>,
|
||||
cancel_token: Option<&CancellationToken>,
|
||||
use_server_limit: bool,
|
||||
on_item: &mut impl FnMut(ClickHouseQueryStreamItem) -> Result<(), String>,
|
||||
) -> Result<(), String> {
|
||||
let max_rows = max_rows.map(|max_rows| max_rows.max(1));
|
||||
let limit = max_rows.map(QueryResultLimit::Limited).unwrap_or(QueryResultLimit::Unlimited);
|
||||
let limit = if use_server_limit {
|
||||
max_rows.map(QueryResultLimit::Limited).unwrap_or(QueryResultLimit::Unlimited)
|
||||
} else {
|
||||
QueryResultLimit::Unlimited
|
||||
};
|
||||
let url = build_query_url_with_format(
|
||||
&client.base_url,
|
||||
Some(database),
|
||||
database,
|
||||
limit,
|
||||
QueryResultFormat::JsonCompactEachRowWithNamesAndTypes,
|
||||
client.extra_params.as_deref(),
|
||||
@@ -339,7 +388,7 @@ pub async fn stream_query_with_max_rows(
|
||||
};
|
||||
|
||||
loop {
|
||||
let chunk = match cancel_token.as_ref() {
|
||||
let chunk = match cancel_token {
|
||||
Some(token) => {
|
||||
tokio::select! {
|
||||
biased;
|
||||
@@ -366,6 +415,28 @@ pub async fn stream_query_with_max_rows(
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn stream_query_with_max_rows(
|
||||
client: &ChClient,
|
||||
database: &str,
|
||||
sql: &str,
|
||||
max_rows: Option<usize>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
mut on_item: impl FnMut(ClickHouseQueryStreamItem) -> Result<(), String>,
|
||||
) -> Result<(), String> {
|
||||
let max_rows = max_rows.map(|max_rows| max_rows.max(1));
|
||||
let result =
|
||||
stream_query_once(client, Some(database), sql, max_rows, cancel_token.as_ref(), true, &mut on_item).await;
|
||||
match result {
|
||||
Err(error) if max_rows.is_some() && is_readonly_result_limit_error(&error) => {
|
||||
log::warn!(
|
||||
"[clickhouse] server-side stream limit rejected by readonly profile; retrying with client-side limit"
|
||||
);
|
||||
stream_query_once(client, Some(database), sql, max_rows, cancel_token.as_ref(), false, &mut on_item).await
|
||||
}
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
|
||||
fn query_result_row_limit(max_rows: Option<usize>) -> usize {
|
||||
max_rows.unwrap_or(MAX_ROWS).max(1)
|
||||
}
|
||||
@@ -675,6 +746,73 @@ pub async fn execute_query_with_max_rows(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
async fn read_http_request(socket: &mut tokio::net::TcpStream) -> String {
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let mut request = Vec::new();
|
||||
let mut chunk = [0_u8; 4096];
|
||||
let header_end = loop {
|
||||
if let Some(index) = request.windows(4).position(|window| window == b"\r\n\r\n") {
|
||||
break index + 4;
|
||||
}
|
||||
let bytes_read = socket.read(&mut chunk).await.unwrap();
|
||||
assert!(bytes_read > 0, "request ended before headers were complete");
|
||||
request.extend_from_slice(&chunk[..bytes_read]);
|
||||
};
|
||||
let headers = String::from_utf8(request[..header_end].to_vec()).unwrap();
|
||||
let content_length = headers
|
||||
.lines()
|
||||
.find_map(|line| {
|
||||
let (name, value) = line.split_once(':')?;
|
||||
name.eq_ignore_ascii_case("content-length").then(|| value.trim().parse::<usize>().unwrap())
|
||||
})
|
||||
.unwrap_or(0);
|
||||
while request.len() < header_end + content_length {
|
||||
let bytes_read = socket.read(&mut chunk).await.unwrap();
|
||||
assert!(bytes_read > 0, "request ended before body was complete");
|
||||
request.extend_from_slice(&chunk[..bytes_read]);
|
||||
}
|
||||
String::from_utf8(request).unwrap()
|
||||
}
|
||||
|
||||
async fn write_http_response(socket: &mut tokio::net::TcpStream, status: &str, body: &str) {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
let response = format!(
|
||||
"HTTP/1.1 {status}\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
socket.write_all(response.as_bytes()).await.unwrap();
|
||||
}
|
||||
|
||||
async fn spawn_readonly_fallback_server(
|
||||
success_body: &'static str,
|
||||
) -> (String, tokio::task::JoinHandle<Vec<String>>) {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let mut targets = Vec::new();
|
||||
for request_index in 0..2 {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let request = read_http_request(&mut socket).await;
|
||||
targets.push(request.lines().next().unwrap().split_whitespace().nth(1).unwrap().to_string());
|
||||
if request_index == 0 {
|
||||
write_http_response(
|
||||
&mut socket,
|
||||
"500 Internal Server Error",
|
||||
"Code: 164. DB::Exception: Cannot modify 'max_result_rows' setting in readonly mode. (READONLY)",
|
||||
)
|
||||
.await;
|
||||
} else {
|
||||
write_http_response(&mut socket, "200 OK", success_body).await;
|
||||
}
|
||||
}
|
||||
targets
|
||||
});
|
||||
|
||||
(format!("http://{address}"), server)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_url_for_result_sets_adds_row_limit_break_settings() {
|
||||
let url = build_query_url(
|
||||
@@ -690,6 +828,70 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recognizes_only_readonly_result_limit_errors() {
|
||||
assert!(is_readonly_result_limit_error(
|
||||
"Code: 164. DB::Exception: Cannot modify 'max_result_rows' setting in readonly mode. (READONLY)"
|
||||
));
|
||||
assert!(is_readonly_result_limit_error(
|
||||
"Code: 164. DB::Exception: Cannot modify 'result_overflow_mode' setting in readonly mode. (READONLY)"
|
||||
));
|
||||
assert!(!is_readonly_result_limit_error(
|
||||
"Code: 164. DB::Exception: Cannot modify 'max_execution_time' setting in readonly mode. (READONLY)"
|
||||
));
|
||||
assert!(!is_readonly_result_limit_error("Code: 62. DB::Exception: Syntax error near max_result_rows"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn readonly_result_query_retries_without_server_limit_and_truncates_client_side() {
|
||||
let (base_url, server) = spawn_readonly_fallback_server("[\"number\"]\n[\"UInt64\"]\n[0]\n[1]\n[2]\n").await;
|
||||
let client = ChClient::new(&base_url, None, None, Duration::from_secs(5));
|
||||
|
||||
let result =
|
||||
execute_query_with_max_rows(&client, "default", "SELECT number FROM numbers(10)", Some(2)).await.unwrap();
|
||||
let targets = server.await.unwrap();
|
||||
|
||||
assert_eq!(result.columns, vec!["number"]);
|
||||
assert_eq!(result.column_types, vec!["UInt64"]);
|
||||
assert_eq!(result.rows, vec![vec![serde_json::json!(0)], vec![serde_json::json!(1)]]);
|
||||
assert!(result.truncated);
|
||||
assert!(targets[0].contains("max_result_rows=3"));
|
||||
assert!(targets[0].contains("result_overflow_mode=break"));
|
||||
assert!(!targets[1].contains("max_result_rows"));
|
||||
assert!(!targets[1].contains("result_overflow_mode"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn readonly_stream_query_retries_without_server_limit_and_stops_client_side() {
|
||||
let (base_url, server) = spawn_readonly_fallback_server("[\"number\"]\n[\"UInt64\"]\n[0]\n[1]\n[2]\n").await;
|
||||
let client = ChClient::new(&base_url, None, None, Duration::from_secs(5));
|
||||
let mut items = Vec::new();
|
||||
|
||||
stream_query_with_max_rows(&client, "default", "SELECT number FROM numbers(10)", Some(2), None, |item| {
|
||||
items.push(item);
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let targets = server.await.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
items,
|
||||
vec![
|
||||
ClickHouseQueryStreamItem::Columns {
|
||||
columns: vec!["number".to_string()],
|
||||
column_types: vec!["UInt64".to_string()],
|
||||
},
|
||||
ClickHouseQueryStreamItem::Row(vec![serde_json::json!(0)]),
|
||||
ClickHouseQueryStreamItem::Row(vec![serde_json::json!(1)]),
|
||||
]
|
||||
);
|
||||
assert!(targets[0].contains("max_result_rows=2"));
|
||||
assert!(targets[0].contains("result_overflow_mode=break"));
|
||||
assert!(!targets[1].contains("max_result_rows"));
|
||||
assert!(!targets[1].contains("result_overflow_mode"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_url_appends_custom_url_params() {
|
||||
// 用户在「URL 参数」填 dialect_type=ANSI,应追加到 query string 末尾
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use dbx_core::db::clickhouse_driver::{
|
||||
execute_query_with_max_rows, stream_query_with_max_rows, ChClient, ClickHouseQueryStreamItem,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DBX_LIVE_CLICKHOUSE_* env vars pointing at a readonly=1 ClickHouse HTTP user"]
|
||||
async fn live_clickhouse_readonly_user_uses_client_side_result_limits() {
|
||||
let host = std::env::var("DBX_LIVE_CLICKHOUSE_HOST").expect("DBX_LIVE_CLICKHOUSE_HOST");
|
||||
let port = std::env::var("DBX_LIVE_CLICKHOUSE_PORT").expect("DBX_LIVE_CLICKHOUSE_PORT");
|
||||
let user = std::env::var("DBX_LIVE_CLICKHOUSE_USER").expect("DBX_LIVE_CLICKHOUSE_USER");
|
||||
let password = std::env::var("DBX_LIVE_CLICKHOUSE_PASSWORD").expect("DBX_LIVE_CLICKHOUSE_PASSWORD");
|
||||
let database = std::env::var("DBX_LIVE_CLICKHOUSE_DATABASE").expect("DBX_LIVE_CLICKHOUSE_DATABASE");
|
||||
let client = ChClient::new(&format!("http://{host}:{port}"), Some(user), Some(password), Duration::from_secs(30));
|
||||
|
||||
let result = execute_query_with_max_rows(&client, &database, "SELECT number FROM numbers(10)", Some(2))
|
||||
.await
|
||||
.expect("readonly ClickHouse query should fall back to client-side limiting");
|
||||
assert_eq!(result.columns, vec!["number"]);
|
||||
assert_eq!(result.column_types, vec!["UInt64"]);
|
||||
assert_eq!(result.rows, vec![vec![serde_json::json!(0)], vec![serde_json::json!(1)]]);
|
||||
assert!(result.truncated);
|
||||
|
||||
let mut columns = Vec::new();
|
||||
let mut rows = Vec::new();
|
||||
stream_query_with_max_rows(&client, &database, "SELECT number FROM numbers(10)", Some(2), None, |item| {
|
||||
match item {
|
||||
ClickHouseQueryStreamItem::Columns { columns: names, column_types } => {
|
||||
columns = names.into_iter().zip(column_types).collect();
|
||||
}
|
||||
ClickHouseQueryStreamItem::Row(row) => rows.push(row),
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.expect("readonly ClickHouse stream should fall back to client-side limiting");
|
||||
assert_eq!(columns, vec![("number".to_string(), "UInt64".to_string())]);
|
||||
assert_eq!(rows, vec![vec![serde_json::json!(0)], vec![serde_json::json!(1)]]);
|
||||
}
|
||||
Reference in new issue
Block a user