Files
dbx/crates/dbx-core/src/db/clickhouse_driver.rs
T

459 lines
17 KiB
Rust

use reqwest::{Certificate, Client as HttpClient};
use serde::Deserialize;
use std::fs;
use std::time::{Duration, Instant};
use super::{http_client_builder, with_connection_timeout};
use crate::query::MAX_ROWS;
use crate::sql::starts_with_executable_sql_keyword;
use crate::types::{ColumnInfo, DatabaseInfo, IndexInfo, QueryResult, TableInfo};
pub struct ChClient {
http: HttpClient,
base_url: String,
username: Option<String>,
password: Option<String>,
}
impl ChClient {
pub fn new(url: &str, username: Option<String>, password: Option<String>, timeout: Duration) -> Self {
let http = http_client_builder(timeout).build().unwrap_or_else(|_| HttpClient::new());
Self { http, base_url: url.trim_end_matches('/').to_string(), username, password }
}
pub fn new_with_ca_cert(
url: &str,
username: Option<String>,
password: Option<String>,
ca_cert_path: Option<&str>,
timeout: Duration,
) -> Result<Self, String> {
let mut builder = http_client_builder(timeout);
if let Some(path) = ca_cert_path.map(str::trim).filter(|path| !path.is_empty()) {
let path = expand_cert_path(path);
let cert_bytes =
fs::read(&path).map_err(|e| format!("Failed to read ClickHouse CA certificate at {path}: {e}"))?;
let cert = Certificate::from_pem(&cert_bytes)
.or_else(|_| Certificate::from_der(&cert_bytes))
.map_err(|e| format!("Failed to parse ClickHouse CA certificate at {path}: {e}"))?;
builder = builder.add_root_certificate(cert);
}
let http = builder.build().map_err(|e| format!("Failed to configure ClickHouse HTTP client: {e}"))?;
Ok(Self { http, base_url: url.trim_end_matches('/').to_string(), username, password })
}
}
fn expand_cert_path(path: &str) -> String {
let home = || std::env::var(if cfg!(windows) { "USERPROFILE" } else { "HOME" }).ok();
if path == "~" || path.starts_with("~/") || path.starts_with("~\\") {
if let Some(home) = home() {
return format!("{}{}", home, &path[1..]);
}
}
if let Some(rest) = path.strip_prefix("$HOME") {
if let Some(home) = home() {
return format!("{home}{rest}");
}
}
if let Some(rest) = path.strip_prefix("${HOME}") {
if let Some(home) = home() {
return format!("{home}{rest}");
}
}
if let Some(rest) = path.strip_prefix("%USERPROFILE%") {
if let Ok(home) = std::env::var("USERPROFILE") {
return format!("{home}{rest}");
}
}
path.to_string()
}
impl Clone for ChClient {
fn clone(&self) -> Self {
Self {
http: self.http.clone(),
base_url: self.base_url.clone(),
username: self.username.clone(),
password: self.password.clone(),
}
}
}
#[derive(Deserialize)]
struct ChJsonResult {
meta: Vec<ChColumn>,
data: Vec<Vec<serde_json::Value>>,
#[serde(default)]
#[allow(dead_code)]
rows: usize,
}
#[derive(Deserialize)]
struct ChColumn {
name: String,
#[serde(rename = "type")]
_type: String,
}
enum QueryResultLimit {
Unlimited,
Limited(usize),
}
fn build_query_url(base_url: &str, database: Option<&str>, limit: QueryResultLimit) -> String {
let mut url = format!("{}/?default_format=JSONCompact", base_url);
if let Some(db) = database {
url.push_str(&format!("&database={db}"));
}
if let QueryResultLimit::Limited(max_rows) = limit {
url.push_str(&format!("&max_result_rows={max_rows}&result_overflow_mode=break"));
}
url
}
fn build_request(client: &ChClient, req: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
match (&client.username, &client.password) {
(Some(u), Some(p)) if !u.is_empty() => req.basic_auth(u, Some(p)),
(Some(u), None) if !u.is_empty() => req.basic_auth(u, None::<&str>),
_ => req,
}
}
async fn ch_query(client: &ChClient, sql: &str, database: Option<&str>) -> Result<ChJsonResult, String> {
ch_query_with_limit(client, sql, database, QueryResultLimit::Unlimited).await
}
async fn ch_query_with_limit(
client: &ChClient,
sql: &str,
database: Option<&str>,
limit: QueryResultLimit,
) -> Result<ChJsonResult, String> {
let url = build_query_url(&client.base_url, database, limit);
log::info!("[clickhouse] query url={url} user={:?} has_pass={}", client.username, client.password.is_some());
let req = build_request(client, client.http.post(&url).body(sql.to_string()));
let resp = req.send().await.map_err(|e| format!("ClickHouse request failed: {e}"))?;
log::info!("[clickhouse] response status={}", resp.status());
if !resp.status().is_success() {
let body = resp.text().await.unwrap_or_default();
log::error!("[clickhouse] error body: {body}");
return Err(format!("ClickHouse error: {body}"));
}
resp.json::<ChJsonResult>().await.map_err(|e| format!("ClickHouse parse error: {e}"))
}
fn query_result_row_limit(max_rows: Option<usize>) -> usize {
max_rows.unwrap_or(MAX_ROWS).max(1)
}
fn clickhouse_literal(value: &str) -> String {
value.replace('\\', "\\\\").replace('\'', "\\'")
}
fn json_value_as_u64(value: Option<&serde_json::Value>) -> Option<u64> {
value.and_then(|value| value.as_u64().or_else(|| value.as_str().and_then(|text| text.parse::<u64>().ok())))
}
fn split_clickhouse_expression_list(value: &str) -> Vec<String> {
let trimmed = value.trim();
if trimmed.is_empty() {
return Vec::new();
}
let inner = trimmed.strip_prefix("tuple(").and_then(|rest| rest.strip_suffix(')')).unwrap_or(trimmed);
let mut items = Vec::new();
let mut start = 0;
let mut depth = 0i32;
let mut quote: Option<char> = None;
let mut escaped = false;
for (idx, ch) in inner.char_indices() {
if let Some(q) = quote {
if escaped {
escaped = false;
} else if ch == '\\' {
escaped = true;
} else if ch == q {
quote = None;
}
continue;
}
match ch {
'\'' | '"' | '`' => quote = Some(ch),
'(' | '[' | '{' => depth += 1,
')' | ']' | '}' => depth = (depth - 1).max(0),
',' if depth == 0 => {
let item = inner[start..idx].trim();
if !item.is_empty() {
items.push(item.to_string());
}
start = idx + ch.len_utf8();
}
_ => {}
}
}
let item = inner[start..].trim();
if !item.is_empty() {
items.push(item.to_string());
}
items
}
fn clickhouse_index_from_skipping_row(row: &[serde_json::Value]) -> IndexInfo {
let index_type = row.get(2).and_then(|v| v.as_str()).unwrap_or("").to_string();
let granularity = json_value_as_u64(row.get(3));
IndexInfo {
name: row.first().and_then(|v| v.as_str()).unwrap_or("").to_string(),
columns: split_clickhouse_expression_list(row.get(1).and_then(|v| v.as_str()).unwrap_or("")),
is_unique: false,
is_primary: false,
filter: None,
index_type: Some(match granularity {
Some(value) => format!("{index_type} GRANULARITY {value}"),
None => index_type,
}),
included_columns: None,
comment: None,
}
}
fn limited_query_result(result: ChJsonResult, execution_time_ms: u128, max_rows: Option<usize>) -> QueryResult {
let columns: Vec<String> = result.meta.iter().map(|c| c.name.clone()).collect();
let column_types: Vec<String> = result.meta.iter().map(|c| c._type.clone()).collect();
let mut rows = result.data;
let row_limit = query_result_row_limit(max_rows);
let truncated = rows.len() > row_limit;
if truncated {
rows.truncate(row_limit);
}
QueryResult {
columns,
column_types,
column_sortables: vec![],
rows,
affected_rows: 0,
execution_time_ms,
truncated,
session_id: None,
has_more: false,
}
}
pub async fn test_connection(client: &ChClient, timeout: Duration) -> Result<(), String> {
let url = format!("{}/?query=SELECT%201", client.base_url);
let req = build_request(client, client.http.get(&url));
let resp = with_connection_timeout("ClickHouse", timeout, async {
req.send().await.map_err(|e| format!("ClickHouse connection failed: {e}"))
})
.await?;
if !resp.status().is_success() {
let body = resp.text().await.unwrap_or_default();
return Err(format!("ClickHouse error: {body}"));
}
Ok(())
}
pub async fn list_databases(client: &ChClient) -> Result<Vec<DatabaseInfo>, String> {
let result = ch_query(client, "SELECT name FROM system.databases ORDER BY name", None).await?;
Ok(result.data.iter().map(|row| DatabaseInfo { name: row[0].as_str().unwrap_or("").to_string() }).collect())
}
pub async fn list_tables(client: &ChClient, database: &str) -> Result<Vec<TableInfo>, String> {
let sql = format!(
"SELECT name, engine FROM system.tables WHERE database = '{}' ORDER BY name",
database.replace('\'', "\\'")
);
let result = ch_query(client, &sql, Some(database)).await?;
Ok(result
.data
.iter()
.map(|row| {
let engine = row.get(1).and_then(|v| v.as_str()).unwrap_or("");
let table_type = if engine.contains("View") { "VIEW" } else { "BASE TABLE" };
TableInfo {
name: row[0].as_str().unwrap_or("").to_string(),
table_type: table_type.to_string(),
comment: None,
parent_schema: None,
parent_name: None,
}
})
.collect())
}
pub async fn get_columns(client: &ChClient, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
let sql = format!(
"SELECT name, type, default_kind, default_expression, is_in_primary_key, is_in_partition_key, comment \
FROM system.columns WHERE database = '{}' AND table = '{}' ORDER BY position",
database.replace('\'', "\\'"),
table.replace('\'', "\\'")
);
let result = ch_query(client, &sql, Some(database)).await?;
Ok(result
.data
.iter()
.map(|row| {
let data_type = row.get(1).and_then(|v| v.as_str()).unwrap_or("").to_string();
let is_nullable = data_type.starts_with("Nullable");
let is_pk = row.get(4).and_then(|v| v.as_u64()).unwrap_or(0) == 1;
let is_partition_key = row.get(5).and_then(|v| v.as_u64()).unwrap_or(0) == 1;
let default_kind = row.get(2).and_then(|v| v.as_str()).unwrap_or("");
let default_expr = row.get(3).and_then(|v| v.as_str()).unwrap_or("");
let column_default = if default_kind.is_empty() { None } else { Some(default_expr.to_string()) };
ColumnInfo {
name: row[0].as_str().unwrap_or("").to_string(),
data_type,
is_nullable,
column_default,
is_primary_key: is_pk,
extra: is_partition_key.then(|| "partition_key".to_string()),
comment: row.get(6).and_then(|v| v.as_str()).filter(|value| !value.is_empty()).map(str::to_string),
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
}
})
.collect())
}
pub async fn list_indexes(client: &ChClient, database: &str, table: &str) -> Result<Vec<IndexInfo>, String> {
let database_lit = clickhouse_literal(database);
let table_lit = clickhouse_literal(table);
let primary_sql = format!(
"SELECT primary_key FROM system.tables WHERE database = '{database_lit}' AND name = '{table_lit}' LIMIT 1"
);
let skipping_sql = format!(
"SELECT name, expr, type, granularity \
FROM system.data_skipping_indices \
WHERE database = '{database_lit}' AND table = '{table_lit}' \
ORDER BY name"
);
let mut indexes = Vec::new();
let primary_result = ch_query(client, &primary_sql, Some(database)).await?;
if let Some(primary_key) = primary_result
.data
.first()
.and_then(|row| row.first())
.and_then(|value| value.as_str())
.filter(|value| !value.trim().is_empty())
{
indexes.push(IndexInfo {
name: "PRIMARY".to_string(),
columns: split_clickhouse_expression_list(primary_key),
is_unique: false,
is_primary: true,
filter: None,
index_type: Some("primary".to_string()),
included_columns: None,
comment: None,
});
}
let skipping_result = ch_query(client, &skipping_sql, Some(database)).await?;
indexes.extend(skipping_result.data.iter().map(|row| clickhouse_index_from_skipping_row(row)));
Ok(indexes)
}
pub async fn execute_query(client: &ChClient, database: &str, sql: &str) -> Result<QueryResult, String> {
execute_query_with_max_rows(client, database, sql, None).await
}
pub async fn execute_query_with_max_rows(
client: &ChClient,
database: &str,
sql: &str,
max_rows: Option<usize>,
) -> Result<QueryResult, String> {
let start = Instant::now();
let row_limit = query_result_row_limit(max_rows);
if starts_with_executable_sql_keyword(sql, &["SELECT", "SHOW", "DESCRIBE", "EXPLAIN", "WITH"]) {
let result = ch_query_with_limit(client, sql, Some(database), QueryResultLimit::Limited(row_limit + 1)).await?;
Ok(limited_query_result(result, start.elapsed().as_millis(), Some(row_limit)))
} else {
let url = build_query_url(&client.base_url, Some(database), QueryResultLimit::Unlimited);
let req = build_request(client, client.http.post(&url).body(sql.to_string()));
let resp = req.send().await.map_err(|e| format!("ClickHouse request failed: {e}"))?;
if !resp.status().is_success() {
let body = resp.text().await.unwrap_or_default();
return Err(format!("ClickHouse error: {body}"));
}
Ok(QueryResult {
columns: vec![],
column_types: Vec::new(),
column_sortables: vec![],
rows: vec![],
affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(),
truncated: false,
session_id: None,
has_more: false,
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn query_url_for_result_sets_adds_row_limit_break_settings() {
let url = build_query_url(
"http://localhost:8123",
Some("analytics"),
QueryResultLimit::Limited(crate::query::MAX_ROWS + 1),
);
assert_eq!(
url,
"http://localhost:8123/?default_format=JSONCompact&database=analytics&max_result_rows=10001&result_overflow_mode=break"
);
}
#[test]
fn limited_query_result_truncates_extra_probe_row() {
let result = ChJsonResult {
meta: vec![ChColumn { name: "id".to_string(), _type: "UInt64".to_string() }],
data: (0..=crate::query::MAX_ROWS).map(|value| vec![serde_json::Value::Number(value.into())]).collect(),
rows: crate::query::MAX_ROWS + 1,
};
let result = limited_query_result(result, 12, None);
assert_eq!(result.columns, vec!["id"]);
assert_eq!(result.rows.len(), crate::query::MAX_ROWS);
assert_eq!(result.execution_time_ms, 12);
assert!(result.truncated);
}
#[test]
fn splits_clickhouse_expression_lists_without_splitting_function_args() {
assert_eq!(split_clickhouse_expression_list("user_id, cityHash64(email, status), event_time"), {
vec!["user_id".to_string(), "cityHash64(email, status)".to_string(), "event_time".to_string()]
});
assert_eq!(split_clickhouse_expression_list("tuple(user_id, event_time)"), {
vec!["user_id".to_string(), "event_time".to_string()]
});
}
#[test]
fn maps_clickhouse_data_skipping_index_row() {
let row = vec![
serde_json::Value::String("idx_email".to_string()),
serde_json::Value::String("lower(email)".to_string()),
serde_json::Value::String("bloom_filter".to_string()),
serde_json::Value::String("4".to_string()),
];
let index = clickhouse_index_from_skipping_row(&row);
assert_eq!(index.name, "idx_email");
assert_eq!(index.columns, vec!["lower(email)"]);
assert_eq!(index.index_type.as_deref(), Some("bloom_filter GRANULARITY 4"));
assert!(!index.is_primary);
}
}