feat: extract dbx-core crate for web deployment support
Refactor all database drivers, query execution, schema operations, and business logic into a shared `dbx-core` crate. The Tauri desktop app becomes a thin shell delegating to dbx-core, enabling future web and CLI frontends to reuse the same core.
This commit is contained in:
1 parent
7bee4bca55
commit
cf4f25fd52
48 files changed
+15354
-4759
No files matched your search
@@ -0,0 +1,792 @@
|
||||
use futures::StreamExt;
|
||||
use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION, CONTENT_TYPE};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::json;
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, LazyLock};
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stream cancel registry
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static AI_STREAMS: LazyLock<RwLock<HashMap<String, Arc<AtomicBool>>>> =
|
||||
LazyLock::new(|| RwLock::new(HashMap::new()));
|
||||
|
||||
pub async fn register_stream(session_id: &str) -> Arc<AtomicBool> {
|
||||
let cancelled = Arc::new(AtomicBool::new(false));
|
||||
AI_STREAMS
|
||||
.write()
|
||||
.await
|
||||
.insert(session_id.to_string(), cancelled.clone());
|
||||
cancelled
|
||||
}
|
||||
|
||||
pub async fn cancel_stream(session_id: &str) -> bool {
|
||||
if let Some(flag) = AI_STREAMS.read().await.get(session_id) {
|
||||
flag.store(true, Ordering::Relaxed);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn unregister_stream(session_id: &str) {
|
||||
AI_STREAMS.write().await.remove(session_id);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum AiProvider {
|
||||
Claude,
|
||||
Openai,
|
||||
Custom,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum AiApiStyle {
|
||||
Completions,
|
||||
Responses,
|
||||
}
|
||||
|
||||
impl Default for AiApiStyle {
|
||||
fn default() -> Self {
|
||||
Self::Completions
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AiConfig {
|
||||
pub provider: AiProvider,
|
||||
#[serde(default)]
|
||||
pub api_key: String,
|
||||
#[serde(default)]
|
||||
pub endpoint: String,
|
||||
#[serde(default)]
|
||||
pub model: String,
|
||||
#[serde(default)]
|
||||
pub api_style: AiApiStyle,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AiMessage {
|
||||
pub role: String,
|
||||
pub content: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AiCompletionRequest {
|
||||
pub config: AiConfig,
|
||||
pub system_prompt: String,
|
||||
pub messages: Vec<AiMessage>,
|
||||
pub max_tokens: Option<u32>,
|
||||
pub temperature: Option<f32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AiStreamChunk {
|
||||
pub session_id: String,
|
||||
pub delta: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_delta: Option<String>,
|
||||
pub done: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AiChatMessage {
|
||||
pub role: String,
|
||||
pub content: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AiConversation {
|
||||
pub id: String,
|
||||
pub title: String,
|
||||
pub connection_name: String,
|
||||
pub database: String,
|
||||
pub messages: Vec<AiChatMessage>,
|
||||
pub created_at: String,
|
||||
pub updated_at: String,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pure helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub fn resolve_endpoint(config: &AiConfig) -> String {
|
||||
let ep = config.endpoint.trim().trim_end_matches('/');
|
||||
if ep.ends_with("/chat/completions")
|
||||
|| ep.ends_with("/responses")
|
||||
|| ep.ends_with("/messages")
|
||||
{
|
||||
return ep.to_string();
|
||||
}
|
||||
match config.provider {
|
||||
AiProvider::Claude => format!("{ep}/messages"),
|
||||
AiProvider::Openai | AiProvider::Custom => {
|
||||
if config.api_style == AiApiStyle::Responses {
|
||||
format!("{ep}/responses")
|
||||
} else {
|
||||
format!("{ep}/chat/completions")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stream_data_payload(line: &str) -> Option<&str> {
|
||||
let line = line.trim();
|
||||
if line.is_empty()
|
||||
|| line.starts_with(':')
|
||||
|| line.starts_with("event:")
|
||||
|| line.starts_with("id:")
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if let Some(data) = line.strip_prefix("data:") {
|
||||
return Some(data.trim_start());
|
||||
}
|
||||
if line.starts_with('{') {
|
||||
return Some(line);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub fn claude_stream_text(event: &serde_json::Value) -> Option<&str> {
|
||||
if event["type"] == "content_block_delta" {
|
||||
return event["delta"]["text"].as_str();
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub fn openai_stream_text(event: &serde_json::Value) -> Option<&str> {
|
||||
event["choices"]
|
||||
.get(0)
|
||||
.and_then(|choice| {
|
||||
choice["delta"]["content"]
|
||||
.as_str()
|
||||
.or_else(|| choice["message"]["content"].as_str())
|
||||
})
|
||||
.or_else(|| event["content"].as_str())
|
||||
.filter(|text| !text.is_empty())
|
||||
}
|
||||
|
||||
pub fn openai_stream_reasoning(event: &serde_json::Value) -> Option<&str> {
|
||||
event["choices"]
|
||||
.get(0)
|
||||
.and_then(|choice| choice["delta"]["reasoning_content"].as_str())
|
||||
.filter(|text| !text.is_empty())
|
||||
}
|
||||
|
||||
pub fn responses_stream_text(event: &serde_json::Value) -> Option<&str> {
|
||||
event["delta"].as_str().filter(|s| !s.is_empty())
|
||||
}
|
||||
|
||||
pub fn extract_error(data: &serde_json::Value) -> Option<String> {
|
||||
data["error"]["message"]
|
||||
.as_str()
|
||||
.or_else(|| data["error"].as_str())
|
||||
.map(ToString::to_string)
|
||||
}
|
||||
|
||||
pub fn build_responses_input(system_prompt: &str, messages: &[AiMessage]) -> serde_json::Value {
|
||||
let mut input = Vec::new();
|
||||
if !system_prompt.is_empty() {
|
||||
input.push(json!({
|
||||
"role": "developer",
|
||||
"content": system_prompt,
|
||||
}));
|
||||
}
|
||||
for m in messages {
|
||||
input.push(json!({
|
||||
"role": m.role,
|
||||
"content": m.content,
|
||||
}));
|
||||
}
|
||||
json!(input)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Validation helper
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn validate_config(config: &AiConfig) -> Result<(), String> {
|
||||
if config.api_key.trim().is_empty() {
|
||||
return Err("API key is required".to_string());
|
||||
}
|
||||
if config.endpoint.trim().is_empty() {
|
||||
return Err("Endpoint is required".to_string());
|
||||
}
|
||||
if config.model.trim().is_empty() {
|
||||
return Err("Model is required".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Non-streaming calls
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub async fn call_claude(
|
||||
client: &reqwest::Client,
|
||||
request: AiCompletionRequest,
|
||||
) -> Result<String, String> {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
headers.insert(
|
||||
"x-api-key",
|
||||
HeaderValue::from_str(&request.config.api_key).map_err(|e| e.to_string())?,
|
||||
);
|
||||
headers.insert("anthropic-version", HeaderValue::from_static("2023-06-01"));
|
||||
|
||||
let body = json!({
|
||||
"model": request.config.model,
|
||||
"max_tokens": request.max_tokens.unwrap_or(2048),
|
||||
"temperature": request.temperature.unwrap_or(0.2),
|
||||
"system": request.system_prompt,
|
||||
"messages": request.messages,
|
||||
});
|
||||
|
||||
let res = client
|
||||
.post(&resolve_endpoint(&request.config))
|
||||
.headers(headers)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("Claude request failed: {e}"))?;
|
||||
|
||||
let status = res.status();
|
||||
let data: serde_json::Value = res.json().await.map_err(|e| e.to_string())?;
|
||||
if !status.is_success() {
|
||||
return Err(extract_error(&data).unwrap_or_else(|| format!("Claude API error: {status}")));
|
||||
}
|
||||
|
||||
Ok(data["content"]
|
||||
.as_array()
|
||||
.and_then(|items| items.iter().find_map(|item| item["text"].as_str()))
|
||||
.unwrap_or_default()
|
||||
.to_string())
|
||||
}
|
||||
|
||||
pub async fn call_openai_compatible(
|
||||
client: &reqwest::Client,
|
||||
request: AiCompletionRequest,
|
||||
) -> Result<String, String> {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
headers.insert(
|
||||
AUTHORIZATION,
|
||||
HeaderValue::from_str(&format!("Bearer {}", request.config.api_key))
|
||||
.map_err(|e| e.to_string())?,
|
||||
);
|
||||
|
||||
let mut messages = vec![json!({ "role": "system", "content": request.system_prompt })];
|
||||
messages.extend(
|
||||
request
|
||||
.messages
|
||||
.iter()
|
||||
.map(|message| json!({ "role": message.role, "content": message.content })),
|
||||
);
|
||||
|
||||
let body = json!({
|
||||
"model": request.config.model,
|
||||
"messages": messages,
|
||||
"max_tokens": request.max_tokens.unwrap_or(2048),
|
||||
"temperature": request.temperature.unwrap_or(0.2),
|
||||
});
|
||||
|
||||
let res = client
|
||||
.post(&resolve_endpoint(&request.config))
|
||||
.headers(headers)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("AI request failed: {e}"))?;
|
||||
|
||||
let status = res.status();
|
||||
let data: serde_json::Value = res.json().await.map_err(|e| e.to_string())?;
|
||||
if !status.is_success() {
|
||||
return Err(extract_error(&data).unwrap_or_else(|| format!("API error: {status}")));
|
||||
}
|
||||
|
||||
Ok(data["choices"][0]["message"]["content"]
|
||||
.as_str()
|
||||
.unwrap_or_default()
|
||||
.to_string())
|
||||
}
|
||||
|
||||
pub async fn call_responses_api(
|
||||
client: &reqwest::Client,
|
||||
request: AiCompletionRequest,
|
||||
) -> Result<String, String> {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
headers.insert(
|
||||
AUTHORIZATION,
|
||||
HeaderValue::from_str(&format!("Bearer {}", request.config.api_key))
|
||||
.map_err(|e| e.to_string())?,
|
||||
);
|
||||
|
||||
let body = json!({
|
||||
"model": request.config.model,
|
||||
"input": build_responses_input(&request.system_prompt, &request.messages),
|
||||
"max_output_tokens": request.max_tokens.unwrap_or(2048),
|
||||
"temperature": request.temperature.unwrap_or(0.2),
|
||||
});
|
||||
|
||||
let res = client
|
||||
.post(&resolve_endpoint(&request.config))
|
||||
.headers(headers)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("AI request failed: {e}"))?;
|
||||
|
||||
let status = res.status();
|
||||
let data: serde_json::Value = res.json().await.map_err(|e| e.to_string())?;
|
||||
if !status.is_success() {
|
||||
return Err(extract_error(&data).unwrap_or_else(|| format!("API error: {status}")));
|
||||
}
|
||||
|
||||
Ok(data["output"]
|
||||
.as_array()
|
||||
.and_then(|items| {
|
||||
items.iter().find_map(|item| {
|
||||
item["content"]
|
||||
.as_array()
|
||||
.and_then(|parts| parts.iter().find_map(|p| p["text"].as_str()))
|
||||
})
|
||||
})
|
||||
.unwrap_or_default()
|
||||
.to_string())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// High-level: test_connection_core / complete
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub async fn test_connection_core(config: &AiConfig) -> Result<String, String> {
|
||||
validate_config(config)?;
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(15))
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let request = AiCompletionRequest {
|
||||
config: config.clone(),
|
||||
system_prompt: String::new(),
|
||||
messages: vec![AiMessage {
|
||||
role: "user".into(),
|
||||
content: "hi".into(),
|
||||
}],
|
||||
max_tokens: Some(1),
|
||||
temperature: Some(0.0),
|
||||
};
|
||||
|
||||
match request.config.provider {
|
||||
AiProvider::Claude => call_claude(&client, request).await,
|
||||
AiProvider::Openai | AiProvider::Custom => {
|
||||
if request.config.api_style == AiApiStyle::Responses {
|
||||
call_responses_api(&client, request).await
|
||||
} else {
|
||||
call_openai_compatible(&client, request).await
|
||||
}
|
||||
}
|
||||
}
|
||||
.map(|_| "OK".to_string())
|
||||
}
|
||||
|
||||
pub async fn complete(request: &AiCompletionRequest) -> Result<String, String> {
|
||||
validate_config(&request.config)?;
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(60))
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
match request.config.provider {
|
||||
AiProvider::Claude => call_claude(&client, request.clone()).await,
|
||||
AiProvider::Openai | AiProvider::Custom => {
|
||||
if request.config.api_style == AiApiStyle::Responses {
|
||||
call_responses_api(&client, request.clone()).await
|
||||
} else {
|
||||
call_openai_compatible(&client, request.clone()).await
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Streaming
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub async fn stream(
|
||||
session_id: &str,
|
||||
request: &AiCompletionRequest,
|
||||
cancelled: &AtomicBool,
|
||||
on_chunk: impl Fn(AiStreamChunk),
|
||||
) -> Result<(), String> {
|
||||
validate_config(&request.config)?;
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(120))
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
match request.config.provider {
|
||||
AiProvider::Claude => {
|
||||
stream_claude(&client, session_id, request, cancelled, &on_chunk).await
|
||||
}
|
||||
AiProvider::Openai | AiProvider::Custom => {
|
||||
if request.config.api_style == AiApiStyle::Responses {
|
||||
stream_responses_api(&client, session_id, request, cancelled, &on_chunk).await
|
||||
} else {
|
||||
stream_openai(&client, session_id, request, cancelled, &on_chunk).await
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn stream_claude(
|
||||
client: &reqwest::Client,
|
||||
session_id: &str,
|
||||
request: &AiCompletionRequest,
|
||||
cancelled: &AtomicBool,
|
||||
on_chunk: &impl Fn(AiStreamChunk),
|
||||
) -> Result<(), String> {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
headers.insert(
|
||||
"x-api-key",
|
||||
HeaderValue::from_str(&request.config.api_key).map_err(|e| e.to_string())?,
|
||||
);
|
||||
headers.insert("anthropic-version", HeaderValue::from_static("2023-06-01"));
|
||||
|
||||
let body = json!({
|
||||
"model": request.config.model,
|
||||
"max_tokens": request.max_tokens.unwrap_or(2048),
|
||||
"temperature": request.temperature.unwrap_or(0.2),
|
||||
"system": request.system_prompt,
|
||||
"messages": request.messages,
|
||||
"stream": true,
|
||||
});
|
||||
|
||||
let res = client
|
||||
.post(&resolve_endpoint(&request.config))
|
||||
.headers(headers)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("Claude request failed: {e}"))?;
|
||||
|
||||
if !res.status().is_success() {
|
||||
let data: serde_json::Value = res.json().await.map_err(|e| e.to_string())?;
|
||||
return Err(extract_error(&data).unwrap_or_else(|| "Claude API error".to_string()));
|
||||
}
|
||||
|
||||
let mut byte_stream = res.bytes_stream();
|
||||
let mut buf = String::new();
|
||||
|
||||
let mut finished = false;
|
||||
while let Some(chunk) = byte_stream.next().await {
|
||||
if cancelled.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
let chunk = chunk.map_err(|e| e.to_string())?;
|
||||
buf.push_str(&String::from_utf8_lossy(&chunk));
|
||||
|
||||
while let Some(pos) = buf.find('\n') {
|
||||
let line = buf[..pos].to_string();
|
||||
buf = buf[pos + 1..].to_string();
|
||||
|
||||
let Some(data) = stream_data_payload(&line) else {
|
||||
continue;
|
||||
};
|
||||
if data == "[DONE]" {
|
||||
finished = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if let Ok(event) = serde_json::from_str::<serde_json::Value>(data) {
|
||||
if let Some(text) = claude_stream_text(&event) {
|
||||
on_chunk(AiStreamChunk {
|
||||
session_id: session_id.to_string(),
|
||||
delta: text.to_string(),
|
||||
reasoning_delta: None,
|
||||
done: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if finished {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
on_chunk(AiStreamChunk {
|
||||
session_id: session_id.to_string(),
|
||||
delta: String::new(),
|
||||
reasoning_delta: None,
|
||||
done: true,
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn stream_openai(
|
||||
client: &reqwest::Client,
|
||||
session_id: &str,
|
||||
request: &AiCompletionRequest,
|
||||
cancelled: &AtomicBool,
|
||||
on_chunk: &impl Fn(AiStreamChunk),
|
||||
) -> Result<(), String> {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
headers.insert(
|
||||
AUTHORIZATION,
|
||||
HeaderValue::from_str(&format!("Bearer {}", request.config.api_key))
|
||||
.map_err(|e| e.to_string())?,
|
||||
);
|
||||
|
||||
let mut messages = vec![json!({ "role": "system", "content": request.system_prompt })];
|
||||
messages.extend(
|
||||
request
|
||||
.messages
|
||||
.iter()
|
||||
.map(|m| json!({ "role": m.role, "content": m.content })),
|
||||
);
|
||||
|
||||
let body = json!({
|
||||
"model": request.config.model,
|
||||
"messages": messages,
|
||||
"max_tokens": request.max_tokens.unwrap_or(2048),
|
||||
"temperature": request.temperature.unwrap_or(0.2),
|
||||
"stream": true,
|
||||
});
|
||||
|
||||
let res = client
|
||||
.post(&resolve_endpoint(&request.config))
|
||||
.headers(headers)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("AI request failed: {e}"))?;
|
||||
|
||||
if !res.status().is_success() {
|
||||
let data: serde_json::Value = res.json().await.map_err(|e| e.to_string())?;
|
||||
return Err(extract_error(&data).unwrap_or_else(|| "API error".to_string()));
|
||||
}
|
||||
|
||||
let mut byte_stream = res.bytes_stream();
|
||||
let mut buf = String::new();
|
||||
|
||||
let mut finished = false;
|
||||
while let Some(chunk) = byte_stream.next().await {
|
||||
if cancelled.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
let chunk = chunk.map_err(|e| e.to_string())?;
|
||||
buf.push_str(&String::from_utf8_lossy(&chunk));
|
||||
|
||||
while let Some(pos) = buf.find('\n') {
|
||||
let line = buf[..pos].to_string();
|
||||
buf = buf[pos + 1..].to_string();
|
||||
|
||||
let Some(data) = stream_data_payload(&line) else {
|
||||
continue;
|
||||
};
|
||||
if data == "[DONE]" {
|
||||
finished = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if let Ok(event) = serde_json::from_str::<serde_json::Value>(data) {
|
||||
if let Some(reasoning) = openai_stream_reasoning(&event) {
|
||||
on_chunk(AiStreamChunk {
|
||||
session_id: session_id.to_string(),
|
||||
delta: String::new(),
|
||||
reasoning_delta: Some(reasoning.to_string()),
|
||||
done: false,
|
||||
});
|
||||
}
|
||||
if let Some(text) = openai_stream_text(&event) {
|
||||
on_chunk(AiStreamChunk {
|
||||
session_id: session_id.to_string(),
|
||||
delta: text.to_string(),
|
||||
reasoning_delta: None,
|
||||
done: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if finished {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
on_chunk(AiStreamChunk {
|
||||
session_id: session_id.to_string(),
|
||||
delta: String::new(),
|
||||
reasoning_delta: None,
|
||||
done: true,
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn stream_responses_api(
|
||||
client: &reqwest::Client,
|
||||
session_id: &str,
|
||||
request: &AiCompletionRequest,
|
||||
cancelled: &AtomicBool,
|
||||
on_chunk: &impl Fn(AiStreamChunk),
|
||||
) -> Result<(), String> {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
headers.insert(
|
||||
AUTHORIZATION,
|
||||
HeaderValue::from_str(&format!("Bearer {}", request.config.api_key))
|
||||
.map_err(|e| e.to_string())?,
|
||||
);
|
||||
|
||||
let body = json!({
|
||||
"model": request.config.model,
|
||||
"input": build_responses_input(&request.system_prompt, &request.messages),
|
||||
"max_output_tokens": request.max_tokens.unwrap_or(2048),
|
||||
"temperature": request.temperature.unwrap_or(0.2),
|
||||
"stream": true,
|
||||
});
|
||||
|
||||
let res = client
|
||||
.post(&resolve_endpoint(&request.config))
|
||||
.headers(headers)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("AI request failed: {e}"))?;
|
||||
|
||||
if !res.status().is_success() {
|
||||
let data: serde_json::Value = res.json().await.map_err(|e| e.to_string())?;
|
||||
return Err(extract_error(&data).unwrap_or_else(|| "API error".to_string()));
|
||||
}
|
||||
|
||||
let mut byte_stream = res.bytes_stream();
|
||||
let mut buf = String::new();
|
||||
|
||||
let mut finished = false;
|
||||
while let Some(chunk) = byte_stream.next().await {
|
||||
if cancelled.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
let chunk = chunk.map_err(|e| e.to_string())?;
|
||||
buf.push_str(&String::from_utf8_lossy(&chunk));
|
||||
|
||||
while let Some(pos) = buf.find('\n') {
|
||||
let line = buf[..pos].to_string();
|
||||
buf = buf[pos + 1..].to_string();
|
||||
|
||||
let Some(data) = stream_data_payload(&line) else {
|
||||
continue;
|
||||
};
|
||||
if data == "[DONE]" {
|
||||
finished = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if let Ok(event) = serde_json::from_str::<serde_json::Value>(data) {
|
||||
if let Some(text) = responses_stream_text(&event) {
|
||||
on_chunk(AiStreamChunk {
|
||||
session_id: session_id.to_string(),
|
||||
delta: text.to_string(),
|
||||
reasoning_delta: None,
|
||||
done: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if finished {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
on_chunk(AiStreamChunk {
|
||||
session_id: session_id.to_string(),
|
||||
delta: String::new(),
|
||||
reasoning_delta: None,
|
||||
done: true,
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Conversation persistence (path-based)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const MAX_CONVERSATIONS: usize = 50;
|
||||
|
||||
pub fn read_conversations(path: &Path) -> Result<Vec<AiConversation>, String> {
|
||||
if !path.exists() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
let json = std::fs::read_to_string(path).map_err(|e| e.to_string())?;
|
||||
serde_json::from_str(&json).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub fn write_conversations(path: &Path, conversations: &[AiConversation]) -> Result<(), String> {
|
||||
let json = serde_json::to_string(conversations).map_err(|e| e.to_string())?;
|
||||
std::fs::write(path, json).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub fn save_conversation(path: &Path, conversation: AiConversation) -> Result<(), String> {
|
||||
let mut conversations = read_conversations(path)?;
|
||||
if let Some(pos) = conversations.iter().position(|c| c.id == conversation.id) {
|
||||
conversations[pos] = conversation;
|
||||
} else {
|
||||
conversations.insert(0, conversation);
|
||||
conversations.truncate(MAX_CONVERSATIONS);
|
||||
}
|
||||
write_conversations(path, &conversations)
|
||||
}
|
||||
|
||||
pub fn load_conversations(path: &Path) -> Result<Vec<AiConversation>, String> {
|
||||
read_conversations(path)
|
||||
}
|
||||
|
||||
pub fn delete_conversation(path: &Path, id: &str) -> Result<(), String> {
|
||||
let conversations: Vec<AiConversation> = read_conversations(path)?
|
||||
.into_iter()
|
||||
.filter(|c| c.id != id)
|
||||
.collect();
|
||||
write_conversations(path, &conversations)
|
||||
}
|
||||
|
||||
pub fn save_config(path: &Path, config: &AiConfig) -> Result<(), String> {
|
||||
let json = serde_json::to_string_pretty(config).map_err(|e| e.to_string())?;
|
||||
std::fs::write(path, json).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub fn load_config(path: &Path) -> Result<Option<AiConfig>, String> {
|
||||
if !path.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
let json = std::fs::read_to_string(path).map_err(|e| e.to_string())?;
|
||||
serde_json::from_str(&json).map(Some).map_err(|e| e.to_string())
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use crate::db;
|
||||
use crate::db::ssh_tunnel::TunnelManager;
|
||||
use crate::models::connection::{ConnectionConfig, DatabaseType};
|
||||
use crate::query_cancel::RunningQueries;
|
||||
|
||||
pub fn expand_tilde(path: &str) -> String {
|
||||
if path == "~" || path.starts_with("~/") {
|
||||
if let Ok(home) = std::env::var(if cfg!(windows) { "USERPROFILE" } else { "HOME" }) {
|
||||
return format!("{}{}", home, &path[1..]);
|
||||
}
|
||||
}
|
||||
path.to_string()
|
||||
}
|
||||
|
||||
pub enum PoolKind {
|
||||
Mysql(sqlx::mysql::MySqlPool, bool),
|
||||
Postgres(sqlx::postgres::PgPool),
|
||||
Sqlite(sqlx::sqlite::SqlitePool),
|
||||
Redis(tokio::sync::Mutex<redis::aio::MultiplexedConnection>),
|
||||
DuckDb(Arc<std::sync::Mutex<duckdb::Connection>>),
|
||||
MongoDb(mongodb::Client),
|
||||
ClickHouse(db::clickhouse_driver::ChClient),
|
||||
SqlServer(Arc<tokio::sync::Mutex<db::sqlserver::SqlServerClient>>),
|
||||
Oracle(Arc<tokio::sync::Mutex<db::oracle_driver::OracleClient>>),
|
||||
Elasticsearch(db::elasticsearch_driver::EsClient),
|
||||
}
|
||||
|
||||
pub struct AppState {
|
||||
pub connections: Mutex<HashMap<String, PoolKind>>,
|
||||
pub configs: Mutex<HashMap<String, ConnectionConfig>>,
|
||||
pub running_queries: RunningQueries,
|
||||
pub tunnels: TunnelManager,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
connections: Mutex::new(HashMap::new()),
|
||||
configs: Mutex::new(HashMap::new()),
|
||||
running_queries: RunningQueries::default(),
|
||||
tunnels: TunnelManager::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_or_create_pool(
|
||||
&self,
|
||||
connection_id: &str,
|
||||
database: Option<&str>,
|
||||
) -> Result<String, String> {
|
||||
let db_type = {
|
||||
let configs = self.configs.lock().await;
|
||||
configs.get(connection_id).map(|c| c.db_type.clone())
|
||||
};
|
||||
|
||||
let is_embedded = matches!(db_type, Some(DatabaseType::Sqlite) | Some(DatabaseType::DuckDb));
|
||||
if is_embedded {
|
||||
return Ok(connection_id.to_string());
|
||||
}
|
||||
|
||||
let is_single_conn = matches!(db_type, Some(DatabaseType::Oracle));
|
||||
let pool_key = if is_single_conn {
|
||||
connection_id.to_string()
|
||||
} else {
|
||||
match database {
|
||||
Some(db) => format!("{connection_id}:{db}"),
|
||||
None => connection_id.to_string(),
|
||||
}
|
||||
};
|
||||
|
||||
let conns = self.connections.lock().await;
|
||||
if conns.contains_key(&pool_key) {
|
||||
return Ok(pool_key);
|
||||
}
|
||||
drop(conns);
|
||||
|
||||
let configs = self.configs.lock().await;
|
||||
let config = configs
|
||||
.get(connection_id)
|
||||
.ok_or("Connection config not found")?
|
||||
.clone();
|
||||
drop(configs);
|
||||
|
||||
let mut db_config = config.clone();
|
||||
if let Some(db) = database {
|
||||
if db_config.db_type != DatabaseType::Oracle {
|
||||
db_config.database = Some(db.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let (host, port) = self.connection_host_port(connection_id, &db_config).await?;
|
||||
let url = connection_url_for_endpoint(&db_config, &host, port);
|
||||
let pool = match db_config.db_type {
|
||||
DatabaseType::Mysql if db_config.needs_bare_mysql() => PoolKind::Mysql(db::mysql::connect_bare(&url).await?, true),
|
||||
DatabaseType::Mysql => PoolKind::Mysql(db::mysql::connect(&url).await?, false),
|
||||
DatabaseType::Doris | DatabaseType::StarRocks => PoolKind::Mysql(db::mysql::connect_bare(&url).await?, true),
|
||||
DatabaseType::Postgres | DatabaseType::Redshift => PoolKind::Postgres(db::postgres::connect(&url).await?),
|
||||
DatabaseType::Sqlite => PoolKind::Sqlite(db::sqlite::connect_path(&expand_tilde(&db_config.host)).await?),
|
||||
DatabaseType::Redis => {
|
||||
let con = db::redis_driver::connect(&url).await?;
|
||||
PoolKind::Redis(tokio::sync::Mutex::new(con))
|
||||
}
|
||||
DatabaseType::DuckDb => {
|
||||
let con = duckdb::Connection::open(&expand_tilde(&db_config.host)).map_err(|e| e.to_string())?;
|
||||
PoolKind::DuckDb(Arc::new(std::sync::Mutex::new(con)))
|
||||
}
|
||||
DatabaseType::MongoDb => {
|
||||
let client = mongodb::Client::with_uri_str(&url).await.map_err(|e| e.to_string())?;
|
||||
PoolKind::MongoDb(client)
|
||||
}
|
||||
DatabaseType::ClickHouse => {
|
||||
let username = if db_config.username.is_empty() { None } else { Some(db_config.username.clone()) };
|
||||
let password = if db_config.password.is_empty() { None } else { Some(db_config.password.clone()) };
|
||||
let client = db::clickhouse_driver::ChClient::new(&url, username, password);
|
||||
db::clickhouse_driver::test_connection(&client).await?;
|
||||
PoolKind::ClickHouse(client)
|
||||
}
|
||||
DatabaseType::SqlServer => {
|
||||
let client = db::sqlserver::connect(
|
||||
&host,
|
||||
port,
|
||||
&db_config.username,
|
||||
&db_config.password,
|
||||
db_config.database.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
PoolKind::SqlServer(Arc::new(tokio::sync::Mutex::new(client)))
|
||||
}
|
||||
DatabaseType::Oracle => {
|
||||
let client = db::oracle_driver::connect(
|
||||
&host,
|
||||
port,
|
||||
db_config.database.as_deref().unwrap_or("ORCL"),
|
||||
&db_config.username, &db_config.password,
|
||||
)
|
||||
.await?;
|
||||
PoolKind::Oracle(Arc::new(tokio::sync::Mutex::new(client)))
|
||||
}
|
||||
DatabaseType::Elasticsearch => {
|
||||
let client = db::elasticsearch_driver::EsClient::new(
|
||||
&url,
|
||||
Some(&db_config.username),
|
||||
Some(&db_config.password),
|
||||
);
|
||||
db::elasticsearch_driver::test_connection(&client).await?;
|
||||
PoolKind::Elasticsearch(client)
|
||||
}
|
||||
};
|
||||
|
||||
self.connections.lock().await.insert(pool_key.clone(), pool);
|
||||
Ok(pool_key)
|
||||
}
|
||||
|
||||
pub async fn connection_host_port(
|
||||
&self,
|
||||
connection_id: &str,
|
||||
config: &ConnectionConfig,
|
||||
) -> Result<(String, u16), String> {
|
||||
if !config.ssh_enabled || config.ssh_host.is_empty() {
|
||||
return Ok((config.host.clone(), config.port));
|
||||
}
|
||||
|
||||
if let Some(local_port) = self.tunnels.local_port(connection_id).await {
|
||||
return Ok(("127.0.0.1".to_string(), local_port));
|
||||
}
|
||||
|
||||
let local_port = self
|
||||
.tunnels
|
||||
.start_tunnel(
|
||||
connection_id,
|
||||
&config.ssh_host,
|
||||
config.ssh_port,
|
||||
&config.ssh_user,
|
||||
&config.ssh_password,
|
||||
&config.ssh_key_path,
|
||||
&config.ssh_key_passphrase,
|
||||
&config.host,
|
||||
config.port,
|
||||
config.ssh_expose_lan,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(("127.0.0.1".to_string(), local_port))
|
||||
}
|
||||
|
||||
pub async fn reconnect_pool(
|
||||
&self,
|
||||
connection_id: &str,
|
||||
database: Option<&str>,
|
||||
) -> Result<String, String> {
|
||||
let is_single_conn = {
|
||||
let configs = self.configs.lock().await;
|
||||
configs.get(connection_id)
|
||||
.map(|c| c.db_type == DatabaseType::Oracle || c.db_type == DatabaseType::Elasticsearch)
|
||||
.unwrap_or(false)
|
||||
};
|
||||
let pool_key = if is_single_conn {
|
||||
connection_id.to_string()
|
||||
} else {
|
||||
match database {
|
||||
Some(db) => format!("{connection_id}:{db}"),
|
||||
None => connection_id.to_string(),
|
||||
}
|
||||
};
|
||||
self.connections.lock().await.remove(&pool_key);
|
||||
self.get_or_create_pool(connection_id, database).await
|
||||
}
|
||||
}
|
||||
|
||||
pub fn connection_url_for_endpoint(config: &ConnectionConfig, host: &str, port: u16) -> String {
|
||||
if host == config.host && port == config.port {
|
||||
config.connection_url()
|
||||
} else {
|
||||
config.connection_url_with_host(host, port)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn redacted_connection_url_for_endpoint(
|
||||
config: &ConnectionConfig,
|
||||
host: &str,
|
||||
port: u16,
|
||||
) -> String {
|
||||
if host == config.host && port == config.port {
|
||||
config.redacted_connection_url()
|
||||
} else {
|
||||
config.redacted_connection_url_with_host(host, port)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,435 @@
|
||||
use crate::models::connection::ConnectionConfig;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub const MAIN_PASSWORD_KEY: &str = "password";
|
||||
pub const SSH_PASSWORD_KEY: &str = "ssh_password";
|
||||
pub const SSH_KEY_PASSPHRASE_KEY: &str = "ssh_key_passphrase";
|
||||
pub const CONNECTION_STRING_KEY: &str = "connection_string";
|
||||
|
||||
pub trait ConnectionSecretStore {
|
||||
fn set_secret(&self, connection_id: &str, key: &str, secret: &str) -> Result<(), String>;
|
||||
fn get_secret(&self, connection_id: &str, key: &str) -> Result<Option<String>, String>;
|
||||
fn delete_secret(&self, connection_id: &str, key: &str) -> Result<(), String>;
|
||||
}
|
||||
|
||||
pub struct FileSecretStore {
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl FileSecretStore {
|
||||
pub fn new(path: PathBuf) -> Self {
|
||||
Self { path }
|
||||
}
|
||||
|
||||
fn read_store(&self) -> HashMap<String, String> {
|
||||
std::fs::read_to_string(&self.path)
|
||||
.ok()
|
||||
.and_then(|json| serde_json::from_str(&json).ok())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn write_store(&self, map: &HashMap<String, String>) -> Result<(), String> {
|
||||
let json = serde_json::to_string_pretty(map).map_err(|e| e.to_string())?;
|
||||
std::fs::write(&self.path, json).map_err(|e| e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl ConnectionSecretStore for FileSecretStore {
|
||||
fn set_secret(&self, connection_id: &str, key: &str, secret: &str) -> Result<(), String> {
|
||||
let mut map = self.read_store();
|
||||
map.insert(secret_account(connection_id, key), secret.to_string());
|
||||
self.write_store(&map)
|
||||
}
|
||||
|
||||
fn get_secret(&self, connection_id: &str, key: &str) -> Result<Option<String>, String> {
|
||||
Ok(self.read_store().get(&secret_account(connection_id, key)).cloned())
|
||||
}
|
||||
|
||||
fn delete_secret(&self, connection_id: &str, key: &str) -> Result<(), String> {
|
||||
let mut map = self.read_store();
|
||||
map.remove(&secret_account(connection_id, key));
|
||||
self.write_store(&map)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn save_connections_to_file(
|
||||
path: &Path,
|
||||
configs: &[ConnectionConfig],
|
||||
store: &dyn ConnectionSecretStore,
|
||||
) -> Result<(), String> {
|
||||
delete_removed_connection_secrets(path, configs, store)?;
|
||||
for config in configs {
|
||||
persist_secret(store, &config.id, MAIN_PASSWORD_KEY, &config.password)?;
|
||||
persist_secret(store, &config.id, SSH_PASSWORD_KEY, &config.ssh_password)?;
|
||||
persist_secret(store, &config.id, SSH_KEY_PASSPHRASE_KEY, &config.ssh_key_passphrase)?;
|
||||
persist_optional_secret(
|
||||
store,
|
||||
&config.id,
|
||||
CONNECTION_STRING_KEY,
|
||||
config.connection_string.as_deref(),
|
||||
)?;
|
||||
}
|
||||
|
||||
write_sanitized_connections(path, configs)
|
||||
}
|
||||
|
||||
pub fn load_connections_from_file(
|
||||
path: &Path,
|
||||
store: &dyn ConnectionSecretStore,
|
||||
) -> Result<Vec<ConnectionConfig>, String> {
|
||||
if !path.exists() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
|
||||
let mut configs = read_connections(path)?;
|
||||
let mut needs_rewrite = false;
|
||||
for config in &mut configs {
|
||||
if config.password.is_empty() {
|
||||
if let Some(secret) = store.get_secret(&config.id, MAIN_PASSWORD_KEY)? {
|
||||
config.password = secret;
|
||||
}
|
||||
} else {
|
||||
store.set_secret(&config.id, MAIN_PASSWORD_KEY, &config.password)?;
|
||||
needs_rewrite = true;
|
||||
}
|
||||
|
||||
if config.ssh_password.is_empty() {
|
||||
if let Some(secret) = store.get_secret(&config.id, SSH_PASSWORD_KEY)? {
|
||||
config.ssh_password = secret;
|
||||
}
|
||||
} else {
|
||||
store.set_secret(&config.id, SSH_PASSWORD_KEY, &config.ssh_password)?;
|
||||
needs_rewrite = true;
|
||||
}
|
||||
|
||||
if config.ssh_key_passphrase.is_empty() {
|
||||
if let Some(secret) = store.get_secret(&config.id, SSH_KEY_PASSPHRASE_KEY)? {
|
||||
config.ssh_key_passphrase = secret;
|
||||
}
|
||||
} else {
|
||||
store.set_secret(&config.id, SSH_KEY_PASSPHRASE_KEY, &config.ssh_key_passphrase)?;
|
||||
needs_rewrite = true;
|
||||
}
|
||||
|
||||
match config
|
||||
.connection_string
|
||||
.as_deref()
|
||||
.filter(|secret| !secret.is_empty())
|
||||
{
|
||||
Some(secret) => {
|
||||
store.set_secret(&config.id, CONNECTION_STRING_KEY, secret)?;
|
||||
needs_rewrite = true;
|
||||
}
|
||||
None => {
|
||||
if let Some(secret) = store.get_secret(&config.id, CONNECTION_STRING_KEY)? {
|
||||
config.connection_string = Some(secret);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if needs_rewrite {
|
||||
write_sanitized_connections(path, &configs)?;
|
||||
}
|
||||
|
||||
Ok(configs)
|
||||
}
|
||||
|
||||
fn delete_removed_connection_secrets(
|
||||
path: &Path,
|
||||
configs: &[ConnectionConfig],
|
||||
store: &dyn ConnectionSecretStore,
|
||||
) -> Result<(), String> {
|
||||
if !path.exists() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let previous = match read_connections(path) {
|
||||
Ok(configs) => configs,
|
||||
Err(_) => return Ok(()),
|
||||
};
|
||||
let current_ids: HashSet<&str> = configs.iter().map(|config| config.id.as_str()).collect();
|
||||
for config in previous {
|
||||
if current_ids.contains(config.id.as_str()) {
|
||||
continue;
|
||||
}
|
||||
store.delete_secret(&config.id, MAIN_PASSWORD_KEY)?;
|
||||
store.delete_secret(&config.id, SSH_PASSWORD_KEY)?;
|
||||
store.delete_secret(&config.id, SSH_KEY_PASSPHRASE_KEY)?;
|
||||
store.delete_secret(&config.id, CONNECTION_STRING_KEY)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn persist_secret(
|
||||
store: &dyn ConnectionSecretStore,
|
||||
connection_id: &str,
|
||||
key: &str,
|
||||
secret: &str,
|
||||
) -> Result<(), String> {
|
||||
if secret.is_empty() {
|
||||
store.delete_secret(connection_id, key)
|
||||
} else {
|
||||
store.set_secret(connection_id, key, secret)
|
||||
}
|
||||
}
|
||||
|
||||
fn persist_optional_secret(
|
||||
store: &dyn ConnectionSecretStore,
|
||||
connection_id: &str,
|
||||
key: &str,
|
||||
secret: Option<&str>,
|
||||
) -> Result<(), String> {
|
||||
match secret.filter(|secret| !secret.is_empty()) {
|
||||
Some(secret) => store.set_secret(connection_id, key, secret),
|
||||
None => store.delete_secret(connection_id, key),
|
||||
}
|
||||
}
|
||||
|
||||
fn read_connections(path: &Path) -> Result<Vec<ConnectionConfig>, String> {
|
||||
let json = std::fs::read_to_string(path).map_err(|e| e.to_string())?;
|
||||
serde_json::from_str(&json).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
fn write_sanitized_connections(path: &Path, configs: &[ConnectionConfig]) -> Result<(), String> {
|
||||
let sanitized = sanitize_connections(configs);
|
||||
let json = serde_json::to_string_pretty(&sanitized).map_err(|e| e.to_string())?;
|
||||
std::fs::write(path, json).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
fn sanitize_connections(configs: &[ConnectionConfig]) -> Vec<ConnectionConfig> {
|
||||
configs
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|mut config| {
|
||||
config.password.clear();
|
||||
config.ssh_password.clear();
|
||||
config.ssh_key_passphrase.clear();
|
||||
config.connection_string = None;
|
||||
config
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn secret_account(connection_id: &str, key: &str) -> String {
|
||||
format!("connection:{connection_id}:{key}")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
load_connections_from_file, save_connections_to_file, ConnectionSecretStore,
|
||||
CONNECTION_STRING_KEY, MAIN_PASSWORD_KEY, SSH_PASSWORD_KEY,
|
||||
};
|
||||
use crate::models::connection::{ConnectionConfig, DatabaseType};
|
||||
use std::cell::RefCell;
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
#[derive(Default)]
|
||||
struct MemorySecretStore {
|
||||
values: RefCell<HashMap<String, String>>,
|
||||
deleted: RefCell<Vec<String>>,
|
||||
}
|
||||
|
||||
impl MemorySecretStore {
|
||||
fn set_existing(&self, connection_id: &str, key: &str, value: &str) {
|
||||
self.values
|
||||
.borrow_mut()
|
||||
.insert(secret_key(connection_id, key), value.to_string());
|
||||
}
|
||||
|
||||
fn get_existing(&self, connection_id: &str, key: &str) -> Option<String> {
|
||||
self.values
|
||||
.borrow()
|
||||
.get(&secret_key(connection_id, key))
|
||||
.cloned()
|
||||
}
|
||||
|
||||
fn was_deleted(&self, connection_id: &str, key: &str) -> bool {
|
||||
self.deleted
|
||||
.borrow()
|
||||
.contains(&secret_key(connection_id, key))
|
||||
}
|
||||
}
|
||||
|
||||
impl ConnectionSecretStore for MemorySecretStore {
|
||||
fn set_secret(&self, connection_id: &str, key: &str, secret: &str) -> Result<(), String> {
|
||||
self.values
|
||||
.borrow_mut()
|
||||
.insert(secret_key(connection_id, key), secret.to_string());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_secret(&self, connection_id: &str, key: &str) -> Result<Option<String>, String> {
|
||||
Ok(self
|
||||
.values
|
||||
.borrow()
|
||||
.get(&secret_key(connection_id, key))
|
||||
.cloned())
|
||||
}
|
||||
|
||||
fn delete_secret(&self, connection_id: &str, key: &str) -> Result<(), String> {
|
||||
self.values
|
||||
.borrow_mut()
|
||||
.remove(&secret_key(connection_id, key));
|
||||
self.deleted
|
||||
.borrow_mut()
|
||||
.push(secret_key(connection_id, key));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn secret_key(connection_id: &str, key: &str) -> String {
|
||||
format!("{connection_id}:{key}")
|
||||
}
|
||||
|
||||
fn temp_connections_file(name: &str) -> std::path::PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!(
|
||||
"dbx-connection-secrets-test-{}-{name}",
|
||||
std::process::id()
|
||||
));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
dir.join("connections.json")
|
||||
}
|
||||
|
||||
fn connection(id: &str, password: &str, ssh_password: &str) -> ConnectionConfig {
|
||||
ConnectionConfig {
|
||||
id: id.to_string(),
|
||||
name: format!("{id} connection"),
|
||||
db_type: DatabaseType::Postgres,
|
||||
driver_profile: None,
|
||||
driver_label: None,
|
||||
url_params: None,
|
||||
host: "localhost".to_string(),
|
||||
port: 5432,
|
||||
username: "postgres".to_string(),
|
||||
password: password.to_string(),
|
||||
database: Some("postgres".to_string()),
|
||||
color: None,
|
||||
ssh_enabled: !ssh_password.is_empty(),
|
||||
ssh_host: String::new(),
|
||||
ssh_port: 22,
|
||||
ssh_user: String::new(),
|
||||
ssh_password: ssh_password.to_string(),
|
||||
ssh_key_path: String::new(),
|
||||
ssh_key_passphrase: String::new(),
|
||||
ssh_expose_lan: false,
|
||||
ssl: false,
|
||||
connection_string: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn read_configs(path: &Path) -> Vec<ConnectionConfig> {
|
||||
let json = std::fs::read_to_string(path).unwrap();
|
||||
serde_json::from_str(&json).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn save_connections_moves_passwords_to_secret_store_and_redacts_file() {
|
||||
let path = temp_connections_file("save-redacts");
|
||||
let store = MemorySecretStore::default();
|
||||
let configs = vec![connection("main", "db-secret", "ssh-secret")];
|
||||
|
||||
save_connections_to_file(&path, &configs, &store).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
store.get_existing("main", MAIN_PASSWORD_KEY).as_deref(),
|
||||
Some("db-secret")
|
||||
);
|
||||
assert_eq!(
|
||||
store.get_existing("main", SSH_PASSWORD_KEY).as_deref(),
|
||||
Some("ssh-secret")
|
||||
);
|
||||
let persisted = read_configs(&path);
|
||||
assert_eq!(persisted[0].password, "");
|
||||
assert_eq!(persisted[0].ssh_password, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_connections_restores_passwords_from_secret_store() {
|
||||
let path = temp_connections_file("load-restores");
|
||||
let store = MemorySecretStore::default();
|
||||
store.set_existing("main", MAIN_PASSWORD_KEY, "db-secret");
|
||||
store.set_existing("main", SSH_PASSWORD_KEY, "ssh-secret");
|
||||
let sanitized = vec![connection("main", "", "")];
|
||||
std::fs::write(&path, serde_json::to_string_pretty(&sanitized).unwrap()).unwrap();
|
||||
|
||||
let loaded = load_connections_from_file(&path, &store).unwrap();
|
||||
|
||||
assert_eq!(loaded[0].password, "db-secret");
|
||||
assert_eq!(loaded[0].ssh_password, "ssh-secret");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_connections_migrates_plaintext_passwords_and_rewrites_sanitized_file() {
|
||||
let path = temp_connections_file("migrates-plaintext");
|
||||
let store = MemorySecretStore::default();
|
||||
let legacy = vec![connection("legacy", "plain-db", "plain-ssh")];
|
||||
std::fs::write(&path, serde_json::to_string_pretty(&legacy).unwrap()).unwrap();
|
||||
|
||||
let loaded = load_connections_from_file(&path, &store).unwrap();
|
||||
|
||||
assert_eq!(loaded[0].password, "plain-db");
|
||||
assert_eq!(loaded[0].ssh_password, "plain-ssh");
|
||||
assert_eq!(
|
||||
store.get_existing("legacy", MAIN_PASSWORD_KEY).as_deref(),
|
||||
Some("plain-db")
|
||||
);
|
||||
assert_eq!(
|
||||
store.get_existing("legacy", SSH_PASSWORD_KEY).as_deref(),
|
||||
Some("plain-ssh")
|
||||
);
|
||||
let persisted = read_configs(&path);
|
||||
assert_eq!(persisted[0].password, "");
|
||||
assert_eq!(persisted[0].ssh_password, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn save_connections_deletes_secrets_for_removed_connections() {
|
||||
let path = temp_connections_file("deletes-removed");
|
||||
let store = MemorySecretStore::default();
|
||||
let previous = vec![connection("old", "", ""), connection("kept", "", "")];
|
||||
std::fs::write(&path, serde_json::to_string_pretty(&previous).unwrap()).unwrap();
|
||||
store.set_existing("old", MAIN_PASSWORD_KEY, "old-db");
|
||||
store.set_existing("old", SSH_PASSWORD_KEY, "old-ssh");
|
||||
store.set_existing("kept", MAIN_PASSWORD_KEY, "kept-db");
|
||||
|
||||
save_connections_to_file(&path, &[connection("kept", "new-db", "")], &store).unwrap();
|
||||
|
||||
assert!(store.was_deleted("old", MAIN_PASSWORD_KEY));
|
||||
assert!(store.was_deleted("old", SSH_PASSWORD_KEY));
|
||||
assert_eq!(
|
||||
store.get_existing("kept", MAIN_PASSWORD_KEY).as_deref(),
|
||||
Some("new-db")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn save_connections_moves_connection_string_to_secret_store_and_restores_it() {
|
||||
let path = temp_connections_file("connection-string");
|
||||
let store = MemorySecretStore::default();
|
||||
let mut config = connection("mongo", "", "");
|
||||
config.db_type = DatabaseType::MongoDb;
|
||||
config.connection_string = Some("mongodb://user:secret@localhost/app".to_string());
|
||||
|
||||
save_connections_to_file(&path, &[config], &store).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.get_existing("mongo", CONNECTION_STRING_KEY)
|
||||
.as_deref(),
|
||||
Some("mongodb://user:secret@localhost/app")
|
||||
);
|
||||
let persisted = read_configs(&path);
|
||||
assert_eq!(persisted[0].connection_string, None);
|
||||
|
||||
let loaded = load_connections_from_file(&path, &store).unwrap();
|
||||
assert_eq!(
|
||||
loaded[0].connection_string.as_deref(),
|
||||
Some("mongodb://user:secret@localhost/app")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
use reqwest::Client as HttpClient;
|
||||
use serde::Deserialize;
|
||||
use std::time::Instant;
|
||||
|
||||
use crate::types::{ColumnInfo, DatabaseInfo, QueryResult, TableInfo};
|
||||
|
||||
pub struct ChClient {
|
||||
http: HttpClient,
|
||||
base_url: String,
|
||||
username: Option<String>,
|
||||
password: Option<String>,
|
||||
}
|
||||
|
||||
impl ChClient {
|
||||
pub fn new(url: &str, username: Option<String>, password: Option<String>) -> Self {
|
||||
Self {
|
||||
http: HttpClient::new(),
|
||||
base_url: url.trim_end_matches('/').to_string(),
|
||||
username,
|
||||
password,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Clone for ChClient {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
http: self.http.clone(),
|
||||
base_url: self.base_url.clone(),
|
||||
username: self.username.clone(),
|
||||
password: self.password.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ChJsonResult {
|
||||
meta: Vec<ChColumn>,
|
||||
data: Vec<Vec<serde_json::Value>>,
|
||||
#[serde(default)]
|
||||
#[allow(dead_code)]
|
||||
rows: usize,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ChColumn {
|
||||
name: String,
|
||||
#[serde(rename = "type")]
|
||||
_type: String,
|
||||
}
|
||||
|
||||
fn build_request(client: &ChClient, req: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
|
||||
match (&client.username, &client.password) {
|
||||
(Some(u), Some(p)) if !u.is_empty() => req.basic_auth(u, Some(p)),
|
||||
(Some(u), None) if !u.is_empty() => req.basic_auth(u, None::<&str>),
|
||||
_ => req,
|
||||
}
|
||||
}
|
||||
|
||||
async fn ch_query(client: &ChClient, sql: &str, database: Option<&str>) -> Result<ChJsonResult, String> {
|
||||
let mut url = format!("{}/?default_format=JSONCompact", client.base_url);
|
||||
if let Some(db) = database {
|
||||
url.push_str(&format!("&database={}", db));
|
||||
}
|
||||
let req = build_request(client, client.http.post(&url).body(sql.to_string()));
|
||||
let resp = req
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("ClickHouse request failed: {e}"))?;
|
||||
if !resp.status().is_success() {
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(format!("ClickHouse error: {body}"));
|
||||
}
|
||||
resp.json::<ChJsonResult>().await.map_err(|e| format!("ClickHouse parse error: {e}"))
|
||||
}
|
||||
|
||||
pub async fn test_connection(client: &ChClient) -> Result<(), String> {
|
||||
let url = format!("{}/ping", client.base_url);
|
||||
let req = build_request(client, client.http.get(&url));
|
||||
req.send().await
|
||||
.map_err(|e| format!("ClickHouse connection failed: {e}"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn list_databases(client: &ChClient) -> Result<Vec<DatabaseInfo>, String> {
|
||||
let result = ch_query(client, "SELECT name FROM system.databases ORDER BY name", None).await?;
|
||||
Ok(result.data.iter().map(|row| {
|
||||
DatabaseInfo { name: row[0].as_str().unwrap_or("").to_string() }
|
||||
}).collect())
|
||||
}
|
||||
|
||||
pub async fn list_tables(client: &ChClient, database: &str) -> Result<Vec<TableInfo>, String> {
|
||||
let sql = format!(
|
||||
"SELECT name, engine FROM system.tables WHERE database = '{}' ORDER BY name",
|
||||
database.replace('\'', "\\'")
|
||||
);
|
||||
let result = ch_query(client, &sql, Some(database)).await?;
|
||||
Ok(result.data.iter().map(|row| {
|
||||
let engine = row.get(1).and_then(|v| v.as_str()).unwrap_or("");
|
||||
let table_type = if engine.contains("View") { "VIEW" } else { "BASE TABLE" };
|
||||
TableInfo {
|
||||
name: row[0].as_str().unwrap_or("").to_string(),
|
||||
table_type: table_type.to_string(),
|
||||
}
|
||||
}).collect())
|
||||
}
|
||||
|
||||
pub async fn get_columns(client: &ChClient, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
|
||||
let sql = format!(
|
||||
"SELECT name, type, default_kind, default_expression, is_in_primary_key \
|
||||
FROM system.columns WHERE database = '{}' AND table = '{}' ORDER BY position",
|
||||
database.replace('\'', "\\'"),
|
||||
table.replace('\'', "\\'")
|
||||
);
|
||||
let result = ch_query(client, &sql, Some(database)).await?;
|
||||
Ok(result.data.iter().map(|row| {
|
||||
let data_type = row.get(1).and_then(|v| v.as_str()).unwrap_or("").to_string();
|
||||
let is_nullable = data_type.starts_with("Nullable");
|
||||
let is_pk = row.get(4).and_then(|v| v.as_u64()).unwrap_or(0) == 1;
|
||||
let default_kind = row.get(2).and_then(|v| v.as_str()).unwrap_or("");
|
||||
let default_expr = row.get(3).and_then(|v| v.as_str()).unwrap_or("");
|
||||
let column_default = if default_kind.is_empty() { None } else { Some(default_expr.to_string()) };
|
||||
ColumnInfo {
|
||||
name: row[0].as_str().unwrap_or("").to_string(),
|
||||
data_type,
|
||||
is_nullable,
|
||||
column_default,
|
||||
is_primary_key: is_pk,
|
||||
extra: None, comment: None,
|
||||
numeric_precision: None,
|
||||
numeric_scale: None,
|
||||
character_maximum_length: None,
|
||||
}
|
||||
}).collect())
|
||||
}
|
||||
|
||||
pub async fn execute_query(client: &ChClient, database: &str, sql: &str) -> Result<QueryResult, String> {
|
||||
let start = Instant::now();
|
||||
let trimmed = sql.trim().to_uppercase();
|
||||
|
||||
if trimmed.starts_with("SELECT")
|
||||
|| trimmed.starts_with("SHOW")
|
||||
|| trimmed.starts_with("DESCRIBE")
|
||||
|| trimmed.starts_with("EXPLAIN")
|
||||
|| trimmed.starts_with("WITH")
|
||||
{
|
||||
let result = ch_query(client, sql, Some(database)).await?;
|
||||
let columns: Vec<String> = result.meta.iter().map(|c| c.name.clone()).collect();
|
||||
Ok(QueryResult {
|
||||
columns,
|
||||
rows: result.data,
|
||||
affected_rows: 0,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
})
|
||||
} else {
|
||||
let url = format!("{}/?default_format=JSONCompact&database={}", client.base_url, database);
|
||||
let req = build_request(client, client.http.post(&url).body(sql.to_string()));
|
||||
let resp = req
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("ClickHouse request failed: {e}"))?;
|
||||
if !resp.status().is_success() {
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(format!("ClickHouse error: {body}"));
|
||||
}
|
||||
Ok(QueryResult {
|
||||
columns: vec![],
|
||||
rows: vec![],
|
||||
affected_rows: 0,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
use reqwest::Client as HttpClient;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::db::mongo_driver::MongoDocumentResult;
|
||||
|
||||
pub struct EsClient {
|
||||
http: HttpClient,
|
||||
base_url: String,
|
||||
auth: Option<(String, String)>,
|
||||
}
|
||||
|
||||
impl EsClient {
|
||||
pub fn new(url: &str, username: Option<&str>, password: Option<&str>) -> Self {
|
||||
let auth = match (username, password) {
|
||||
(Some(u), Some(p)) if !u.is_empty() => Some((u.to_string(), p.to_string())),
|
||||
_ => None,
|
||||
};
|
||||
Self {
|
||||
http: HttpClient::new(),
|
||||
base_url: url.trim_end_matches('/').to_string(),
|
||||
auth,
|
||||
}
|
||||
}
|
||||
|
||||
fn get(&self, path: &str) -> reqwest::RequestBuilder {
|
||||
let req = self.http.get(format!("{}{}", self.base_url, path));
|
||||
self.with_auth(req)
|
||||
}
|
||||
|
||||
fn post(&self, path: &str) -> reqwest::RequestBuilder {
|
||||
let req = self.http.post(format!("{}{}", self.base_url, path));
|
||||
self.with_auth(req)
|
||||
}
|
||||
|
||||
fn put(&self, path: &str) -> reqwest::RequestBuilder {
|
||||
let req = self.http.put(format!("{}{}", self.base_url, path));
|
||||
self.with_auth(req)
|
||||
}
|
||||
|
||||
fn delete(&self, path: &str) -> reqwest::RequestBuilder {
|
||||
let req = self.http.delete(format!("{}{}", self.base_url, path));
|
||||
self.with_auth(req)
|
||||
}
|
||||
|
||||
fn with_auth(&self, req: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
|
||||
if let Some((ref user, ref pass)) = self.auth {
|
||||
req.basic_auth(user, Some(pass))
|
||||
} else {
|
||||
req
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Clone for EsClient {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
http: self.http.clone(),
|
||||
base_url: self.base_url.clone(),
|
||||
auth: self.auth.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn test_connection(client: &EsClient) -> Result<(), String> {
|
||||
let resp = client.get("/")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("Elasticsearch connection failed: {e}"))?;
|
||||
if !resp.status().is_success() {
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(format!("Elasticsearch error: {body}"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CatIndex {
|
||||
index: String,
|
||||
}
|
||||
|
||||
pub async fn list_indices(client: &EsClient) -> Result<Vec<String>, String> {
|
||||
let resp = client.get("/_cat/indices?format=json&h=index")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
|
||||
if !resp.status().is_success() {
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(format!("Elasticsearch error: {body}"));
|
||||
}
|
||||
let indices: Vec<CatIndex> = resp.json().await.map_err(|e| format!("Elasticsearch parse error: {e}"))?;
|
||||
let mut names: Vec<String> = indices
|
||||
.into_iter()
|
||||
.filter(|i| !i.index.starts_with('.'))
|
||||
.map(|i| i.index)
|
||||
.collect();
|
||||
names.sort();
|
||||
Ok(names)
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct SearchResponse {
|
||||
hits: SearchHits,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct SearchHits {
|
||||
total: HitsTotal,
|
||||
hits: Vec<SearchHit>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct HitsTotal {
|
||||
value: u64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct SearchHit {
|
||||
#[serde(rename = "_id")]
|
||||
id: String,
|
||||
#[serde(rename = "_source")]
|
||||
source: serde_json::Value,
|
||||
}
|
||||
|
||||
pub async fn find_documents(
|
||||
client: &EsClient,
|
||||
index: &str,
|
||||
skip: u64,
|
||||
limit: i64,
|
||||
) -> Result<MongoDocumentResult, String> {
|
||||
let body = serde_json::json!({
|
||||
"from": skip,
|
||||
"size": limit,
|
||||
"sort": ["_doc"],
|
||||
});
|
||||
|
||||
let path = format!("/{}/_search", index);
|
||||
let resp = client.post(&path)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(format!("Elasticsearch error: {body}"));
|
||||
}
|
||||
|
||||
let result: SearchResponse = resp.json().await
|
||||
.map_err(|e| format!("Elasticsearch parse error: {e}"))?;
|
||||
|
||||
let documents: Vec<serde_json::Value> = result.hits.hits.into_iter().map(|hit| {
|
||||
let mut doc = match hit.source {
|
||||
serde_json::Value::Object(map) => map,
|
||||
_ => serde_json::Map::new(),
|
||||
};
|
||||
doc.insert("_id".to_string(), serde_json::Value::String(hit.id));
|
||||
serde_json::Value::Object(doc)
|
||||
}).collect();
|
||||
|
||||
Ok(MongoDocumentResult {
|
||||
documents,
|
||||
total: result.hits.total.value,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn insert_document(
|
||||
client: &EsClient,
|
||||
index: &str,
|
||||
doc_json: &str,
|
||||
) -> Result<String, String> {
|
||||
let doc: serde_json::Value = serde_json::from_str(doc_json)
|
||||
.map_err(|e| format!("Invalid JSON: {e}"))?;
|
||||
|
||||
let path = format!("/{}/_doc?refresh=true", index);
|
||||
let resp = client.post(&path)
|
||||
.json(&doc)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(format!("Elasticsearch error: {body}"));
|
||||
}
|
||||
|
||||
let result: serde_json::Value = resp.json().await
|
||||
.map_err(|e| format!("Elasticsearch parse error: {e}"))?;
|
||||
Ok(result["_id"].as_str().unwrap_or("").to_string())
|
||||
}
|
||||
|
||||
pub async fn update_document(
|
||||
client: &EsClient,
|
||||
index: &str,
|
||||
id: &str,
|
||||
doc_json: &str,
|
||||
) -> Result<u64, String> {
|
||||
let doc: serde_json::Value = serde_json::from_str(doc_json)
|
||||
.map_err(|e| format!("Invalid JSON: {e}"))?;
|
||||
|
||||
let path = format!("/{}/_doc/{}?refresh=true", index, id);
|
||||
let resp = client.put(&path)
|
||||
.json(&doc)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(format!("Elasticsearch error: {body}"));
|
||||
}
|
||||
|
||||
Ok(1)
|
||||
}
|
||||
|
||||
pub async fn delete_document(
|
||||
client: &EsClient,
|
||||
index: &str,
|
||||
id: &str,
|
||||
) -> Result<u64, String> {
|
||||
let path = format!("/{}/_doc/{}?refresh=true", index, id);
|
||||
let resp = client.delete(&path)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(format!("Elasticsearch error: {body}"));
|
||||
}
|
||||
|
||||
Ok(1)
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
pub mod clickhouse_driver;
|
||||
pub mod elasticsearch_driver;
|
||||
pub mod mongo_driver;
|
||||
pub mod mysql;
|
||||
pub mod oracle_driver;
|
||||
pub mod postgres;
|
||||
pub mod redis_driver;
|
||||
pub mod sqlite;
|
||||
pub mod sqlserver;
|
||||
pub mod ssh_tunnel;
|
||||
|
||||
// Re-export types so that `db::QueryResult` etc. work within dbx-core
|
||||
pub use crate::types::*;
|
||||
@@ -0,0 +1,121 @@
|
||||
use mongodb::{bson::{doc, Document, Bson}, Client};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MongoDocumentResult {
|
||||
pub documents: Vec<serde_json::Value>,
|
||||
pub total: u64,
|
||||
}
|
||||
|
||||
pub async fn list_databases(client: &Client) -> Result<Vec<String>, String> {
|
||||
client
|
||||
.list_database_names()
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub async fn list_collections(client: &Client, database: &str) -> Result<Vec<String>, String> {
|
||||
client
|
||||
.database(database)
|
||||
.list_collection_names()
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub async fn find_documents(
|
||||
client: &Client,
|
||||
database: &str,
|
||||
collection: &str,
|
||||
skip: u64,
|
||||
limit: i64,
|
||||
) -> Result<MongoDocumentResult, String> {
|
||||
let col = client.database(database).collection::<Document>(collection);
|
||||
|
||||
let total = col.count_documents(doc! {}).await.map_err(|e| e.to_string())?;
|
||||
|
||||
let mut cursor = col
|
||||
.find(doc! {})
|
||||
.skip(skip)
|
||||
.limit(limit)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let mut documents = Vec::new();
|
||||
while cursor.advance().await.map_err(|e| e.to_string())? {
|
||||
let doc = cursor.deserialize_current().map_err(|e| e.to_string())?;
|
||||
let json = bson_to_json(&Bson::Document(doc));
|
||||
documents.push(json);
|
||||
}
|
||||
|
||||
Ok(MongoDocumentResult { documents, total })
|
||||
}
|
||||
|
||||
pub async fn insert_document(
|
||||
client: &Client,
|
||||
database: &str,
|
||||
collection: &str,
|
||||
doc_json: &str,
|
||||
) -> Result<String, String> {
|
||||
let doc: Document = serde_json::from_str(doc_json)
|
||||
.map_err(|e| format!("Invalid JSON: {e}"))?;
|
||||
let col = client.database(database).collection::<Document>(collection);
|
||||
let result = col.insert_one(doc).await.map_err(|e| e.to_string())?;
|
||||
Ok(format!("{}", result.inserted_id))
|
||||
}
|
||||
|
||||
pub async fn update_document(
|
||||
client: &Client,
|
||||
database: &str,
|
||||
collection: &str,
|
||||
id: &str,
|
||||
doc_json: &str,
|
||||
) -> Result<u64, String> {
|
||||
let oid = mongodb::bson::oid::ObjectId::parse_str(id)
|
||||
.map_err(|e| format!("Invalid ObjectId: {e}"))?;
|
||||
let new_doc: Document = serde_json::from_str(doc_json)
|
||||
.map_err(|e| format!("Invalid JSON: {e}"))?;
|
||||
let col = client.database(database).collection::<Document>(collection);
|
||||
let result = col
|
||||
.replace_one(doc! { "_id": oid }, new_doc)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(result.modified_count)
|
||||
}
|
||||
|
||||
pub async fn delete_document(
|
||||
client: &Client,
|
||||
database: &str,
|
||||
collection: &str,
|
||||
id: &str,
|
||||
) -> Result<u64, String> {
|
||||
let oid = mongodb::bson::oid::ObjectId::parse_str(id)
|
||||
.map_err(|e| format!("Invalid ObjectId: {e}"))?;
|
||||
let col = client.database(database).collection::<Document>(collection);
|
||||
let result = col
|
||||
.delete_one(doc! { "_id": oid })
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(result.deleted_count)
|
||||
}
|
||||
|
||||
fn bson_to_json(bson: &Bson) -> serde_json::Value {
|
||||
match bson {
|
||||
Bson::Double(v) => serde_json::json!(v),
|
||||
Bson::String(v) => serde_json::Value::String(v.clone()),
|
||||
Bson::Boolean(v) => serde_json::Value::Bool(*v),
|
||||
Bson::Null => serde_json::Value::Null,
|
||||
Bson::Int32(v) => serde_json::json!(v),
|
||||
Bson::Int64(v) => serde_json::json!(v),
|
||||
Bson::ObjectId(oid) => serde_json::Value::String(oid.to_hex()),
|
||||
Bson::DateTime(dt) => serde_json::Value::String(dt.to_string()),
|
||||
Bson::Array(arr) => serde_json::Value::Array(arr.iter().map(bson_to_json).collect()),
|
||||
Bson::Document(doc) => {
|
||||
let mut map = serde_json::Map::new();
|
||||
for (k, v) in doc {
|
||||
map.insert(k.clone(), bson_to_json(v));
|
||||
}
|
||||
serde_json::Value::Object(map)
|
||||
}
|
||||
_ => serde_json::Value::String(format!("{bson}")),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,432 @@
|
||||
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc};
|
||||
use rust_decimal::Decimal;
|
||||
use sqlx::mysql::{MySqlPool, MySqlPoolOptions, MySqlRow};
|
||||
use sqlx::{Column, Executor, Row, TypeInfo, ValueRef};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::types::{ColumnInfo, DatabaseInfo, ForeignKeyInfo, IndexInfo, QueryResult, TableInfo, TriggerInfo};
|
||||
|
||||
fn quote_value(s: &str) -> String {
|
||||
format!("'{}'", s.replace('\\', "\\\\").replace('\'', "\\'"))
|
||||
}
|
||||
|
||||
fn get_str(row: &MySqlRow, idx: usize) -> String {
|
||||
row.try_get::<String, _>(idx)
|
||||
.or_else(|_| row.try_get::<Vec<u8>, _>(idx).map(|b| String::from_utf8_lossy(&b).to_string()))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn get_str_by_name(row: &MySqlRow, name: &str) -> String {
|
||||
row.try_get::<String, _>(name)
|
||||
.or_else(|_| row.try_get::<Vec<u8>, _>(name).map(|b| String::from_utf8_lossy(&b).to_string()))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn get_opt_str(row: &MySqlRow, name: &str) -> Option<String> {
|
||||
row.try_get::<Option<String>, _>(name)
|
||||
.ok()
|
||||
.flatten()
|
||||
.or_else(|| {
|
||||
row.try_get::<Option<Vec<u8>>, _>(name)
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(|b| String::from_utf8_lossy(&b).to_string())
|
||||
})
|
||||
}
|
||||
|
||||
fn numeric_metadata_u64_to_i32(value: Option<u64>) -> Option<i32> {
|
||||
value.and_then(|v| i32::try_from(v).ok())
|
||||
}
|
||||
|
||||
fn numeric_metadata_i64_to_i32(value: Option<i64>) -> Option<i32> {
|
||||
value.and_then(|v| i32::try_from(v).ok())
|
||||
}
|
||||
|
||||
fn numeric_metadata_str_to_i32(value: Option<String>) -> Option<i32> {
|
||||
value.and_then(|v| v.parse::<i64>().ok())
|
||||
.and_then(|v| i32::try_from(v).ok())
|
||||
}
|
||||
|
||||
fn get_opt_i32(row: &MySqlRow, name: &str) -> Option<i32> {
|
||||
if row.try_get_raw(name).map(|v| v.is_null()).unwrap_or(true) {
|
||||
return None;
|
||||
}
|
||||
|
||||
row.try_get::<Option<i32>, _>(name)
|
||||
.ok()
|
||||
.flatten()
|
||||
.or_else(|| numeric_metadata_i64_to_i32(row.try_get::<Option<i64>, _>(name).ok().flatten()))
|
||||
.or_else(|| numeric_metadata_u64_to_i32(row.try_get::<Option<u64>, _>(name).ok().flatten()))
|
||||
.or_else(|| numeric_metadata_str_to_i32(row.try_get::<Option<String>, _>(name).ok().flatten()))
|
||||
.or_else(|| {
|
||||
row.try_get::<Option<Vec<u8>>, _>(name)
|
||||
.ok()
|
||||
.flatten()
|
||||
.and_then(|b| String::from_utf8(b).ok())
|
||||
.and_then(|v| numeric_metadata_str_to_i32(Some(v)))
|
||||
})
|
||||
}
|
||||
|
||||
fn mysql_temporal_to_json_value(row: &MySqlRow, idx: usize) -> Option<serde_json::Value> {
|
||||
if let Ok(v) = row.try_get::<NaiveDateTime, _>(idx) {
|
||||
return Some(serde_json::Value::String(v.to_string()));
|
||||
}
|
||||
if let Ok(v) = row.try_get::<DateTime<Utc>, _>(idx) {
|
||||
return Some(serde_json::Value::String(v.to_rfc3339()));
|
||||
}
|
||||
if let Ok(v) = row.try_get::<NaiveDate, _>(idx) {
|
||||
return Some(serde_json::Value::String(v.to_string()));
|
||||
}
|
||||
if let Ok(v) = row.try_get::<NaiveTime, _>(idx) {
|
||||
return Some(serde_json::Value::String(v.to_string()));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn mysql_value_to_json(row: &MySqlRow, idx: usize, type_name: &str) -> serde_json::Value {
|
||||
if row.try_get_raw(idx).map(|v| v.is_null()).unwrap_or(true) {
|
||||
return serde_json::Value::Null;
|
||||
}
|
||||
|
||||
let upper_type = type_name.to_uppercase();
|
||||
|
||||
if upper_type == "JSON" {
|
||||
if let Ok(v) = row.try_get::<serde_json::Value, _>(idx) {
|
||||
return v;
|
||||
}
|
||||
if let Ok(v) = row.try_get::<String, _>(idx) {
|
||||
return serde_json::from_str::<serde_json::Value>(&v).unwrap_or(serde_json::Value::String(v));
|
||||
}
|
||||
return serde_json::Value::Null;
|
||||
}
|
||||
|
||||
if upper_type == "BOOLEAN" {
|
||||
return row
|
||||
.try_get::<bool, _>(idx)
|
||||
.map(serde_json::Value::Bool)
|
||||
.unwrap_or(serde_json::Value::Null);
|
||||
}
|
||||
|
||||
if upper_type.contains("BIGINT") {
|
||||
return row
|
||||
.try_get::<i64, _>(idx)
|
||||
.map(|v| serde_json::Value::String(v.to_string()))
|
||||
.or_else(|_| {
|
||||
row.try_get::<u64, _>(idx)
|
||||
.map(|v| serde_json::Value::String(v.to_string()))
|
||||
})
|
||||
.unwrap_or(serde_json::Value::Null);
|
||||
}
|
||||
|
||||
if upper_type == "DECIMAL" {
|
||||
return row
|
||||
.try_get::<Decimal, _>(idx)
|
||||
.map(|v: Decimal| serde_json::Value::String(v.to_string()))
|
||||
.unwrap_or(serde_json::Value::Null);
|
||||
}
|
||||
|
||||
if upper_type.starts_with("DATETIME")
|
||||
|| upper_type.starts_with("TIMESTAMP")
|
||||
|| upper_type == "DATE"
|
||||
|| upper_type == "TIME"
|
||||
|| upper_type.starts_with("TIME(")
|
||||
{
|
||||
if let Some(v) = mysql_temporal_to_json_value(row, idx) {
|
||||
return v;
|
||||
}
|
||||
}
|
||||
|
||||
row.try_get::<String, _>(idx)
|
||||
.map(serde_json::Value::String)
|
||||
.or_else(|_| row.try_get::<i64, _>(idx).map(|v| serde_json::Value::Number(v.into())))
|
||||
.or_else(|_| row.try_get::<u64, _>(idx).map(|v| serde_json::Value::Number(v.into())))
|
||||
.or_else(|_| row.try_get::<f64, _>(idx).map(|v| {
|
||||
serde_json::Number::from_f64(v)
|
||||
.map(serde_json::Value::Number)
|
||||
.unwrap_or(serde_json::Value::Null)
|
||||
}))
|
||||
.or_else(|_| row.try_get::<bool, _>(idx).map(serde_json::Value::Bool))
|
||||
.or_else(|_| {
|
||||
row.try_get::<Vec<u8>, _>(idx)
|
||||
.map(|b| serde_json::Value::String(String::from_utf8_lossy(&b).to_string()))
|
||||
})
|
||||
.or_else(|e| mysql_temporal_to_json_value(row, idx).ok_or(e))
|
||||
.unwrap_or(serde_json::Value::Null)
|
||||
}
|
||||
|
||||
pub async fn connect(url: &str) -> Result<MySqlPool, String> {
|
||||
MySqlPoolOptions::new()
|
||||
.max_connections(5)
|
||||
.acquire_timeout(Duration::from_secs(10))
|
||||
.idle_timeout(Duration::from_secs(300))
|
||||
.connect(url)
|
||||
.await
|
||||
.map_err(|e| format!("MySQL connection failed: {e}"))
|
||||
}
|
||||
|
||||
pub async fn connect_bare(url: &str) -> Result<MySqlPool, String> {
|
||||
let options: sqlx::mysql::MySqlConnectOptions = url.parse()
|
||||
.map_err(|e: sqlx::Error| format!("Invalid MySQL URL: {e}"))?;
|
||||
let options = options
|
||||
.no_engine_substitution(false)
|
||||
.set_names(false)
|
||||
.pipes_as_concat(false)
|
||||
.timezone(None);
|
||||
MySqlPoolOptions::new()
|
||||
.max_connections(5)
|
||||
.acquire_timeout(Duration::from_secs(10))
|
||||
.idle_timeout(Duration::from_secs(300))
|
||||
.connect_with(options)
|
||||
.await
|
||||
.map_err(|e| format!("MySQL connection failed: {e}"))
|
||||
}
|
||||
|
||||
pub async fn list_databases(pool: &MySqlPool) -> Result<Vec<DatabaseInfo>, String> {
|
||||
let rows: Vec<MySqlRow> = sqlx::raw_sql("SELECT SCHEMA_NAME FROM information_schema.SCHEMATA ORDER BY SCHEMA_NAME")
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(rows.iter().map(|row| DatabaseInfo { name: get_str(row, 0) }).collect())
|
||||
}
|
||||
|
||||
pub async fn list_tables(pool: &MySqlPool, database: &str) -> Result<Vec<TableInfo>, String> {
|
||||
let sql = format!(
|
||||
"SELECT TABLE_NAME, TABLE_TYPE FROM information_schema.TABLES WHERE TABLE_SCHEMA = {} ORDER BY TABLE_NAME",
|
||||
quote_value(database),
|
||||
);
|
||||
let rows: Vec<MySqlRow> = sqlx::raw_sql(&sql)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(rows
|
||||
.iter()
|
||||
.map(|row| TableInfo {
|
||||
name: get_str_by_name(row, "TABLE_NAME"),
|
||||
table_type: get_str_by_name(row, "TABLE_TYPE"),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn get_columns(
|
||||
pool: &MySqlPool,
|
||||
database: &str,
|
||||
table: &str,
|
||||
) -> Result<Vec<ColumnInfo>, String> {
|
||||
let sql = format!(
|
||||
"SELECT c.COLUMN_NAME, c.COLUMN_TYPE, c.IS_NULLABLE, c.COLUMN_DEFAULT, c.EXTRA, c.COLUMN_COMMENT, \
|
||||
CASE WHEN kcu.COLUMN_NAME IS NOT NULL THEN 1 ELSE 0 END AS IS_PK, \
|
||||
c.NUMERIC_PRECISION, c.NUMERIC_SCALE, c.CHARACTER_MAXIMUM_LENGTH \
|
||||
FROM information_schema.COLUMNS c \
|
||||
LEFT JOIN information_schema.KEY_COLUMN_USAGE kcu \
|
||||
ON c.TABLE_SCHEMA = kcu.TABLE_SCHEMA \
|
||||
AND c.TABLE_NAME = kcu.TABLE_NAME \
|
||||
AND c.COLUMN_NAME = kcu.COLUMN_NAME \
|
||||
AND kcu.CONSTRAINT_NAME = 'PRIMARY' \
|
||||
WHERE c.TABLE_SCHEMA = {} AND c.TABLE_NAME = {} \
|
||||
ORDER BY c.ORDINAL_POSITION",
|
||||
quote_value(database),
|
||||
quote_value(table),
|
||||
);
|
||||
let rows: Vec<MySqlRow> = sqlx::raw_sql(&sql)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(rows
|
||||
.iter()
|
||||
.map(|row| ColumnInfo {
|
||||
name: get_str_by_name(row, "COLUMN_NAME"),
|
||||
data_type: get_str_by_name(row, "COLUMN_TYPE"),
|
||||
is_nullable: get_str_by_name(row, "IS_NULLABLE") == "YES",
|
||||
column_default: get_opt_str(row, "COLUMN_DEFAULT"),
|
||||
is_primary_key: row.get::<i32, _>("IS_PK") == 1,
|
||||
extra: get_opt_str(row, "EXTRA"),
|
||||
comment: get_opt_str(row, "COLUMN_COMMENT").filter(|s| !s.is_empty()),
|
||||
numeric_precision: get_opt_i32(row, "NUMERIC_PRECISION"),
|
||||
numeric_scale: get_opt_i32(row, "NUMERIC_SCALE"),
|
||||
character_maximum_length: get_opt_i32(row, "CHARACTER_MAXIMUM_LENGTH"),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn execute_query(pool: &MySqlPool, sql: &str, bare: bool) -> Result<QueryResult, String> {
|
||||
let start = Instant::now();
|
||||
let trimmed = sql.trim().to_uppercase();
|
||||
|
||||
if trimmed.starts_with("SELECT") || trimmed.starts_with("SHOW") || trimmed.starts_with("DESCRIBE") || trimmed.starts_with("EXPLAIN") {
|
||||
if bare {
|
||||
let rows: Vec<MySqlRow> = sqlx::raw_sql(sql)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let (columns, column_types) = if let Some(first) = rows.first() {
|
||||
let cols: Vec<String> = first.columns().iter().map(|c| c.name().to_string()).collect();
|
||||
let types: Vec<String> = first.columns().iter().map(|c| c.type_info().name().to_string()).collect();
|
||||
(cols, types)
|
||||
} else {
|
||||
(vec![], vec![])
|
||||
};
|
||||
|
||||
let result_rows: Vec<Vec<serde_json::Value>> = rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
(0..row.len())
|
||||
.map(|i| mysql_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(QueryResult {
|
||||
columns,
|
||||
rows: result_rows,
|
||||
affected_rows: 0,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
})
|
||||
} else {
|
||||
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
|
||||
let columns: Vec<String> = desc.columns().iter().map(|c| c.name().to_string()).collect();
|
||||
let column_types: Vec<String> = desc.columns().iter().map(|c| c.type_info().name().to_string()).collect();
|
||||
|
||||
let rows: Vec<MySqlRow> = sqlx::query(sql)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let result_rows: Vec<Vec<serde_json::Value>> = rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
(0..row.len())
|
||||
.map(|i| mysql_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(QueryResult {
|
||||
columns,
|
||||
rows: result_rows,
|
||||
affected_rows: 0,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
})
|
||||
}
|
||||
} else {
|
||||
let result = sqlx::raw_sql(sql)
|
||||
.execute(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(QueryResult {
|
||||
columns: vec![],
|
||||
rows: vec![],
|
||||
affected_rows: result.rows_affected(),
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn list_indexes(pool: &MySqlPool, database: &str, table: &str) -> Result<Vec<IndexInfo>, String> {
|
||||
let sql = format!(
|
||||
"SELECT INDEX_NAME, GROUP_CONCAT(COLUMN_NAME ORDER BY SEQ_IN_INDEX) AS columns, \
|
||||
MIN(NON_UNIQUE) = 0 AS is_unique, INDEX_NAME = 'PRIMARY' AS is_primary, \
|
||||
INDEX_TYPE \
|
||||
FROM information_schema.STATISTICS \
|
||||
WHERE TABLE_SCHEMA = {} AND TABLE_NAME = {} \
|
||||
GROUP BY INDEX_NAME, INDEX_TYPE \
|
||||
ORDER BY INDEX_NAME",
|
||||
quote_value(database),
|
||||
quote_value(table),
|
||||
);
|
||||
let rows: Vec<MySqlRow> = sqlx::raw_sql(&sql)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
let cols_str = get_str_by_name(row, "columns");
|
||||
IndexInfo {
|
||||
name: get_str_by_name(row, "INDEX_NAME"),
|
||||
columns: cols_str.split(',').filter(|s| !s.is_empty()).map(|s| s.to_string()).collect(),
|
||||
is_unique: row.get::<bool, _>("is_unique"),
|
||||
is_primary: row.get::<bool, _>("is_primary"),
|
||||
filter: None,
|
||||
index_type: Some(get_str_by_name(row, "INDEX_TYPE")),
|
||||
included_columns: None,
|
||||
comment: None,
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn list_foreign_keys(pool: &MySqlPool, database: &str, table: &str) -> Result<Vec<ForeignKeyInfo>, String> {
|
||||
let sql = format!(
|
||||
"SELECT kcu.CONSTRAINT_NAME, kcu.COLUMN_NAME, \
|
||||
kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME \
|
||||
FROM information_schema.KEY_COLUMN_USAGE kcu \
|
||||
WHERE kcu.TABLE_SCHEMA = {} AND kcu.TABLE_NAME = {} \
|
||||
AND kcu.REFERENCED_TABLE_NAME IS NOT NULL \
|
||||
ORDER BY kcu.CONSTRAINT_NAME",
|
||||
quote_value(database),
|
||||
quote_value(table),
|
||||
);
|
||||
let rows: Vec<MySqlRow> = sqlx::raw_sql(&sql)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(rows
|
||||
.iter()
|
||||
.map(|row| ForeignKeyInfo {
|
||||
name: get_str_by_name(row, "CONSTRAINT_NAME"),
|
||||
column: get_str_by_name(row, "COLUMN_NAME"),
|
||||
ref_table: get_str_by_name(row, "REFERENCED_TABLE_NAME"),
|
||||
ref_column: get_str_by_name(row, "REFERENCED_COLUMN_NAME"),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn list_triggers(pool: &MySqlPool, database: &str, table: &str) -> Result<Vec<TriggerInfo>, String> {
|
||||
let sql = format!(
|
||||
"SELECT TRIGGER_NAME, EVENT_MANIPULATION, ACTION_TIMING \
|
||||
FROM information_schema.TRIGGERS \
|
||||
WHERE TRIGGER_SCHEMA = {} AND EVENT_OBJECT_TABLE = {} \
|
||||
ORDER BY TRIGGER_NAME",
|
||||
quote_value(database),
|
||||
quote_value(table),
|
||||
);
|
||||
let rows: Vec<MySqlRow> = sqlx::raw_sql(&sql)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(rows
|
||||
.iter()
|
||||
.map(|row| TriggerInfo {
|
||||
name: get_str_by_name(row, "TRIGGER_NAME"),
|
||||
event: get_str_by_name(row, "EVENT_MANIPULATION"),
|
||||
timing: get_str_by_name(row, "ACTION_TIMING"),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn numeric_metadata_accepts_unsigned_information_schema_values() {
|
||||
assert_eq!(numeric_metadata_u64_to_i32(Some(65)), Some(65));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numeric_metadata_ignores_values_outside_frontend_range() {
|
||||
assert_eq!(numeric_metadata_u64_to_i32(Some(i32::MAX as u64 + 1)), None);
|
||||
assert_eq!(numeric_metadata_u64_to_i32(None), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
use oracle_rs::{Config, Connection};
|
||||
use std::time::Instant;
|
||||
|
||||
use crate::types::{ColumnInfo, DatabaseInfo, ForeignKeyInfo, IndexInfo, QueryResult, TableInfo, TriggerInfo};
|
||||
|
||||
pub type OracleClient = Connection;
|
||||
|
||||
pub async fn connect(host: &str, port: u16, service: &str, user: &str, pass: &str) -> Result<OracleClient, String> {
|
||||
let config = Config::new(host, port, service, user, pass);
|
||||
Connection::connect_with_config(config)
|
||||
.await
|
||||
.map_err(|e| format!("Oracle connection failed: {e}"))
|
||||
}
|
||||
|
||||
fn value_to_json(val: &oracle_rs::Value) -> serde_json::Value {
|
||||
match val {
|
||||
oracle_rs::Value::Null => serde_json::Value::Null,
|
||||
oracle_rs::Value::String(s) => serde_json::Value::String(s.clone()),
|
||||
oracle_rs::Value::Integer(n) => serde_json::Value::Number((*n).into()),
|
||||
oracle_rs::Value::Float(f) => serde_json::Number::from_f64(*f)
|
||||
.map(serde_json::Value::Number)
|
||||
.unwrap_or(serde_json::Value::Null),
|
||||
oracle_rs::Value::Boolean(b) => serde_json::Value::Bool(*b),
|
||||
oracle_rs::Value::Json(v) => v.clone(),
|
||||
_ => serde_json::Value::String(format!("{val:?}")),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn list_databases(conn: &OracleClient) -> Result<Vec<DatabaseInfo>, String> {
|
||||
let result = conn.query(
|
||||
"SELECT username FROM all_users ORDER BY username",
|
||||
&[],
|
||||
).await.map_err(|e| e.to_string())?;
|
||||
Ok(result.rows.iter().map(|row| {
|
||||
DatabaseInfo { name: row.get_string(0).unwrap_or("").to_string() }
|
||||
}).collect())
|
||||
}
|
||||
|
||||
pub async fn list_schemas(conn: &OracleClient) -> Result<Vec<String>, String> {
|
||||
let result = conn.query(
|
||||
"SELECT username FROM all_users ORDER BY username",
|
||||
&[],
|
||||
).await.map_err(|e| e.to_string())?;
|
||||
Ok(result.rows.iter().map(|row| {
|
||||
row.get_string(0).unwrap_or("").to_string()
|
||||
}).collect())
|
||||
}
|
||||
|
||||
pub async fn list_tables(conn: &OracleClient, schema: &str) -> Result<Vec<TableInfo>, String> {
|
||||
let sql = format!(
|
||||
"SELECT table_name, 'TABLE' AS table_type FROM all_tables WHERE owner = '{s}' \
|
||||
UNION ALL \
|
||||
SELECT view_name, 'VIEW' FROM all_views WHERE owner = '{s}' \
|
||||
ORDER BY 1",
|
||||
s = schema.replace('\'', "''")
|
||||
);
|
||||
let result = conn.query(&sql, &[]).await.map_err(|e| e.to_string())?;
|
||||
Ok(result.rows.iter().map(|row| {
|
||||
TableInfo {
|
||||
name: row.get_string(0).unwrap_or("").to_string(),
|
||||
table_type: row.get_string(1).unwrap_or("TABLE").to_string(),
|
||||
}
|
||||
}).collect())
|
||||
}
|
||||
|
||||
pub async fn get_columns(conn: &OracleClient, schema: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
|
||||
let s = schema.replace('\'', "''");
|
||||
let t = table.replace('\'', "''");
|
||||
|
||||
let pk_result = conn.query(
|
||||
&format!(
|
||||
"SELECT cols.COLUMN_NAME FROM ALL_CONS_COLUMNS cols \
|
||||
JOIN ALL_CONSTRAINTS cons ON cols.CONSTRAINT_NAME = cons.CONSTRAINT_NAME AND cols.OWNER = cons.OWNER \
|
||||
WHERE cons.CONSTRAINT_TYPE = 'P' AND cons.OWNER = '{s}' AND cons.TABLE_NAME = '{t}'"
|
||||
),
|
||||
&[],
|
||||
).await.map_err(|e| e.to_string())?;
|
||||
let pk_names: std::collections::HashSet<String> = pk_result.rows.iter()
|
||||
.filter_map(|row| row.get_string(0).map(|s| s.to_string()))
|
||||
.collect();
|
||||
|
||||
let col_result = conn.query(
|
||||
&format!(
|
||||
"SELECT COLUMN_NAME, DATA_TYPE, NULLABLE, DATA_PRECISION, DATA_SCALE, DATA_LENGTH, CHAR_LENGTH \
|
||||
FROM ALL_TAB_COLUMNS \
|
||||
WHERE OWNER = '{s}' AND TABLE_NAME = '{t}' \
|
||||
ORDER BY COLUMN_ID"
|
||||
),
|
||||
&[],
|
||||
).await.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(col_result.rows.iter().map(|row| {
|
||||
let name = row.get_string(0).unwrap_or("").to_string();
|
||||
let base = row.get_string(1).unwrap_or("").to_string();
|
||||
let data_len = row.get_i64(5).map(|v| v as i32);
|
||||
let char_len = row.get_i64(6).map(|v| v as i32);
|
||||
let num_prec = row.get_i64(3).map(|v| v as i32);
|
||||
let num_scale = row.get_i64(4).map(|v| v as i32);
|
||||
let data_type = match base.to_uppercase().as_str() {
|
||||
"VARCHAR2" | "NVARCHAR2" | "CHAR" | "NCHAR" => {
|
||||
let len = char_len.or(data_len);
|
||||
match len {
|
||||
Some(n) => format!("{base}({n})"),
|
||||
None => base,
|
||||
}
|
||||
}
|
||||
"NUMBER" => match (num_prec, num_scale) {
|
||||
(Some(p), Some(s)) if s > 0 => format!("NUMBER({p},{s})"),
|
||||
(Some(p), _) if p > 0 => format!("NUMBER({p})"),
|
||||
_ => "NUMBER".to_string(),
|
||||
},
|
||||
"RAW" => match data_len {
|
||||
Some(n) => format!("RAW({n})"),
|
||||
None => "RAW".to_string(),
|
||||
},
|
||||
_ => base,
|
||||
};
|
||||
ColumnInfo {
|
||||
is_primary_key: pk_names.contains(&name),
|
||||
name,
|
||||
data_type,
|
||||
is_nullable: row.get_string(2).unwrap_or("N") == "Y",
|
||||
column_default: None,
|
||||
extra: None, comment: None,
|
||||
numeric_precision: num_prec,
|
||||
numeric_scale: num_scale,
|
||||
character_maximum_length: char_len,
|
||||
}
|
||||
}).collect())
|
||||
}
|
||||
|
||||
pub async fn list_indexes(conn: &OracleClient, schema: &str, table: &str) -> Result<Vec<IndexInfo>, String> {
|
||||
let sql = format!(
|
||||
"SELECT i.INDEX_NAME, \
|
||||
LISTAGG(ic.COLUMN_NAME, ',') WITHIN GROUP (ORDER BY ic.COLUMN_POSITION) AS columns, \
|
||||
i.UNIQUENESS, \
|
||||
CASE WHEN c.CONSTRAINT_TYPE = 'P' THEN 1 ELSE 0 END AS IS_PK, \
|
||||
i.INDEX_TYPE \
|
||||
FROM ALL_INDEXES i \
|
||||
JOIN ALL_IND_COLUMNS ic ON i.INDEX_NAME = ic.INDEX_NAME AND i.OWNER = ic.INDEX_OWNER AND i.TABLE_OWNER = ic.TABLE_OWNER \
|
||||
LEFT JOIN ALL_CONSTRAINTS c ON i.INDEX_NAME = c.INDEX_NAME AND i.TABLE_OWNER = c.OWNER \
|
||||
AND c.CONSTRAINT_TYPE = 'P' \
|
||||
WHERE i.TABLE_OWNER = '{s}' AND i.TABLE_NAME = '{t}' \
|
||||
GROUP BY i.INDEX_NAME, i.UNIQUENESS, c.CONSTRAINT_TYPE, i.INDEX_TYPE \
|
||||
ORDER BY i.INDEX_NAME",
|
||||
s = schema.replace('\'', "''"), t = table.replace('\'', "''")
|
||||
);
|
||||
let result = conn.query(&sql, &[]).await.map_err(|e| e.to_string())?;
|
||||
Ok(result.rows.iter().map(|row| {
|
||||
let cols_str = row.get_string(1).unwrap_or("");
|
||||
IndexInfo {
|
||||
name: row.get_string(0).unwrap_or("").to_string(),
|
||||
columns: cols_str.split(',').filter(|s| !s.is_empty()).map(|s| s.to_string()).collect(),
|
||||
is_unique: row.get_string(2).unwrap_or("") == "UNIQUE",
|
||||
is_primary: row.get_i64(3).unwrap_or(0) == 1,
|
||||
filter: None,
|
||||
index_type: row.get_string(4).map(|s| s.to_string()),
|
||||
included_columns: None,
|
||||
comment: None,
|
||||
}
|
||||
}).collect())
|
||||
}
|
||||
|
||||
pub async fn list_foreign_keys(conn: &OracleClient, schema: &str, table: &str) -> Result<Vec<ForeignKeyInfo>, String> {
|
||||
let sql = format!(
|
||||
"SELECT c.CONSTRAINT_NAME, cc.COLUMN_NAME, rc.TABLE_NAME, rcc.COLUMN_NAME \
|
||||
FROM ALL_CONSTRAINTS c \
|
||||
JOIN ALL_CONS_COLUMNS cc ON c.CONSTRAINT_NAME = cc.CONSTRAINT_NAME AND c.OWNER = cc.OWNER \
|
||||
JOIN ALL_CONSTRAINTS rc ON c.R_CONSTRAINT_NAME = rc.CONSTRAINT_NAME AND c.R_OWNER = rc.OWNER \
|
||||
JOIN ALL_CONS_COLUMNS rcc ON rc.CONSTRAINT_NAME = rcc.CONSTRAINT_NAME AND rc.OWNER = rcc.OWNER \
|
||||
WHERE c.CONSTRAINT_TYPE = 'R' AND c.OWNER = '{s}' AND c.TABLE_NAME = '{t}' \
|
||||
ORDER BY c.CONSTRAINT_NAME",
|
||||
s = schema.replace('\'', "''"), t = table.replace('\'', "''")
|
||||
);
|
||||
let result = conn.query(&sql, &[]).await.map_err(|e| e.to_string())?;
|
||||
Ok(result.rows.iter().map(|row| {
|
||||
ForeignKeyInfo {
|
||||
name: row.get_string(0).unwrap_or("").to_string(),
|
||||
column: row.get_string(1).unwrap_or("").to_string(),
|
||||
ref_table: row.get_string(2).unwrap_or("").to_string(),
|
||||
ref_column: row.get_string(3).unwrap_or("").to_string(),
|
||||
}
|
||||
}).collect())
|
||||
}
|
||||
|
||||
pub async fn list_triggers(conn: &OracleClient, schema: &str, table: &str) -> Result<Vec<TriggerInfo>, String> {
|
||||
let sql = format!(
|
||||
"SELECT TRIGGER_NAME, TRIGGERING_EVENT, TRIGGER_TYPE \
|
||||
FROM ALL_TRIGGERS \
|
||||
WHERE OWNER = '{s}' AND TABLE_NAME = '{t}' \
|
||||
ORDER BY TRIGGER_NAME",
|
||||
s = schema.replace('\'', "''"), t = table.replace('\'', "''")
|
||||
);
|
||||
let result = conn.query(&sql, &[]).await.map_err(|e| e.to_string())?;
|
||||
Ok(result.rows.iter().map(|row| {
|
||||
TriggerInfo {
|
||||
name: row.get_string(0).unwrap_or("").to_string(),
|
||||
event: row.get_string(1).unwrap_or("").to_string(),
|
||||
timing: row.get_string(2).unwrap_or("").to_string(),
|
||||
}
|
||||
}).collect())
|
||||
}
|
||||
|
||||
pub async fn execute_query(conn: &OracleClient, sql: &str) -> Result<QueryResult, String> {
|
||||
let start = Instant::now();
|
||||
let sql = sql.trim().trim_end_matches(';');
|
||||
let trimmed = sql.to_uppercase();
|
||||
|
||||
if trimmed.starts_with("SELECT")
|
||||
|| trimmed.starts_with("WITH")
|
||||
|| trimmed.starts_with("SHOW")
|
||||
|| trimmed.starts_with("DESCRIBE")
|
||||
|| trimmed.starts_with("EXPLAIN")
|
||||
{
|
||||
let result = conn.query(sql, &[]).await.map_err(|e| e.to_string())?;
|
||||
let columns: Vec<String> = result.columns.iter().map(|c| c.name.clone()).collect();
|
||||
let rows: Vec<Vec<serde_json::Value>> = result.rows.iter().map(|row| {
|
||||
(0..columns.len()).map(|i| {
|
||||
row.get(i)
|
||||
.map(|v| value_to_json(v))
|
||||
.unwrap_or(serde_json::Value::Null)
|
||||
}).collect()
|
||||
}).collect();
|
||||
|
||||
Ok(QueryResult {
|
||||
columns,
|
||||
rows,
|
||||
affected_rows: 0,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
})
|
||||
} else {
|
||||
match conn.execute(sql, &[]).await {
|
||||
Ok(result) => {
|
||||
let _ = conn.commit().await;
|
||||
Ok(QueryResult {
|
||||
columns: vec![],
|
||||
rows: vec![],
|
||||
affected_rows: result.rows_affected,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
})
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = e.to_string();
|
||||
if msg.contains("Server rejected") || msg.contains("closed the connection") {
|
||||
Err("Operation failed (connection closed) — possibly a constraint violation (foreign key, unique, or check constraint).".to_string())
|
||||
} else {
|
||||
Err(msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,376 @@
|
||||
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc};
|
||||
use rust_decimal::Decimal;
|
||||
use sqlx::postgres::{PgPool, PgPoolOptions, PgRow};
|
||||
use sqlx::{Column, Executor, Row, TypeInfo, ValueRef};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::types::{ColumnInfo, DatabaseInfo, ForeignKeyInfo, IndexInfo, QueryResult, TableInfo, TriggerInfo};
|
||||
|
||||
fn pg_temporal_to_json_value(row: &PgRow, idx: usize) -> Option<serde_json::Value> {
|
||||
if let Ok(v) = row.try_get::<DateTime<Utc>, _>(idx) {
|
||||
return Some(serde_json::Value::String(v.to_rfc3339()));
|
||||
}
|
||||
if let Ok(v) = row.try_get::<NaiveDateTime, _>(idx) {
|
||||
return Some(serde_json::Value::String(v.to_string()));
|
||||
}
|
||||
if let Ok(v) = row.try_get::<NaiveDate, _>(idx) {
|
||||
return Some(serde_json::Value::String(v.to_string()));
|
||||
}
|
||||
if let Ok(v) = row.try_get::<NaiveTime, _>(idx) {
|
||||
return Some(serde_json::Value::String(v.to_string()));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn pg_value_to_json(row: &PgRow, idx: usize, type_name: &str) -> serde_json::Value {
|
||||
if row.try_get_raw(idx).map(|v| v.is_null()).unwrap_or(true) {
|
||||
return serde_json::Value::Null;
|
||||
}
|
||||
|
||||
let upper = type_name.to_uppercase();
|
||||
|
||||
if upper == "JSON" || upper == "JSONB" {
|
||||
if let Ok(v) = row.try_get::<serde_json::Value, _>(idx) {
|
||||
return v;
|
||||
}
|
||||
if let Ok(v) = row.try_get::<String, _>(idx) {
|
||||
return serde_json::from_str::<serde_json::Value>(&v).unwrap_or(serde_json::Value::String(v));
|
||||
}
|
||||
return serde_json::Value::Null;
|
||||
}
|
||||
|
||||
if upper == "BOOL" {
|
||||
return row
|
||||
.try_get::<bool, _>(idx)
|
||||
.map(serde_json::Value::Bool)
|
||||
.unwrap_or(serde_json::Value::Null);
|
||||
}
|
||||
|
||||
if upper.contains("TIMESTAMP")
|
||||
|| upper == "DATE"
|
||||
|| upper == "TIME"
|
||||
|| upper == "TIMETZ"
|
||||
|| upper.contains("INTERVAL")
|
||||
{
|
||||
if let Some(v) = pg_temporal_to_json_value(row, idx) {
|
||||
return v;
|
||||
}
|
||||
}
|
||||
|
||||
if upper == "NUMERIC" || upper == "DECIMAL" || upper == "MONEY" {
|
||||
return row
|
||||
.try_get::<Decimal, _>(idx)
|
||||
.map(|v: Decimal| serde_json::Value::String(v.to_string()))
|
||||
.unwrap_or(serde_json::Value::Null);
|
||||
}
|
||||
|
||||
row.try_get::<String, _>(idx)
|
||||
.map(serde_json::Value::String)
|
||||
.or_else(|_| {
|
||||
row.try_get::<i64, _>(idx)
|
||||
.map(|v| serde_json::Value::Number(v.into()))
|
||||
})
|
||||
.or_else(|_| {
|
||||
row.try_get::<i32, _>(idx)
|
||||
.map(|v| serde_json::Value::Number(v.into()))
|
||||
})
|
||||
.or_else(|_| {
|
||||
row.try_get::<f64, _>(idx).map(|v| {
|
||||
serde_json::Number::from_f64(v)
|
||||
.map(serde_json::Value::Number)
|
||||
.unwrap_or(serde_json::Value::Null)
|
||||
})
|
||||
})
|
||||
.or_else(|_| row.try_get::<bool, _>(idx).map(serde_json::Value::Bool))
|
||||
.or_else(|e| pg_temporal_to_json_value(row, idx).ok_or(e))
|
||||
.unwrap_or(serde_json::Value::Null)
|
||||
}
|
||||
|
||||
pub async fn connect(url: &str) -> Result<PgPool, String> {
|
||||
PgPoolOptions::new()
|
||||
.max_connections(5)
|
||||
.acquire_timeout(Duration::from_secs(10))
|
||||
.idle_timeout(Duration::from_secs(300))
|
||||
.connect(url)
|
||||
.await
|
||||
.map_err(|e| format!("PostgreSQL connection failed: {e}"))
|
||||
}
|
||||
|
||||
pub async fn list_databases(pool: &PgPool) -> Result<Vec<DatabaseInfo>, String> {
|
||||
let rows: Vec<PgRow> =
|
||||
sqlx::query("SELECT datname FROM pg_database WHERE datistemplate = false ORDER BY datname")
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(rows
|
||||
.iter()
|
||||
.map(|row| DatabaseInfo {
|
||||
name: row.get::<String, _>("datname"),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn list_tables(pool: &PgPool, schema: &str) -> Result<Vec<TableInfo>, String> {
|
||||
let rows: Vec<PgRow> = sqlx::query(
|
||||
"SELECT table_name, table_type FROM information_schema.tables \
|
||||
WHERE table_schema = $1 ORDER BY table_name",
|
||||
)
|
||||
.bind(schema)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(rows
|
||||
.iter()
|
||||
.map(|row| TableInfo {
|
||||
name: row.get::<String, _>("table_name"),
|
||||
table_type: row.get::<String, _>("table_type"),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn list_schemas(pool: &PgPool) -> Result<Vec<String>, String> {
|
||||
let rows: Vec<PgRow> = sqlx::query(
|
||||
"SELECT schema_name FROM information_schema.schemata \
|
||||
WHERE schema_name NOT IN ('information_schema', 'pg_catalog', 'pg_toast') \
|
||||
AND schema_name NOT LIKE 'pg_toast_temp_%' \
|
||||
AND schema_name NOT LIKE 'pg_temp_%' \
|
||||
ORDER BY schema_name",
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(rows
|
||||
.iter()
|
||||
.map(|row| row.get::<String, _>("schema_name"))
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn get_columns(
|
||||
pool: &PgPool,
|
||||
schema: &str,
|
||||
table: &str,
|
||||
) -> Result<Vec<ColumnInfo>, String> {
|
||||
let rows: Vec<PgRow> = sqlx::query(
|
||||
"SELECT a.attname AS column_name, \
|
||||
format_type(a.atttypid, a.atttypmod) AS full_type, \
|
||||
NOT a.attnotnull AS is_nullable, \
|
||||
pg_get_expr(ad.adbin, ad.adrelid) AS column_default, \
|
||||
EXISTS ( \
|
||||
SELECT 1 FROM pg_constraint co \
|
||||
JOIN pg_index i ON i.indrelid = co.conrelid AND co.conindid = i.indexrelid \
|
||||
WHERE co.conrelid = a.attrelid AND co.contype = 'p' \
|
||||
AND a.attnum = ANY(i.indkey) \
|
||||
) AS is_pk, \
|
||||
col_description(a.attrelid, a.attnum) AS column_comment, \
|
||||
CASE WHEN t.typname = 'numeric' AND a.atttypmod > 0 \
|
||||
THEN ((a.atttypmod - 4) >> 16) & 65535 ELSE NULL END AS numeric_precision, \
|
||||
CASE WHEN t.typname = 'numeric' AND a.atttypmod > 0 \
|
||||
THEN (a.atttypmod - 4) & 65535 ELSE NULL END AS numeric_scale, \
|
||||
CASE WHEN t.typname IN ('varchar', 'bpchar') AND a.atttypmod > 0 \
|
||||
THEN a.atttypmod - 4 ELSE NULL END AS character_maximum_length \
|
||||
FROM pg_attribute a \
|
||||
JOIN pg_type t ON t.oid = a.atttypid \
|
||||
LEFT JOIN pg_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum \
|
||||
WHERE a.attrelid = ($1 || '.' || $2)::regclass \
|
||||
AND a.attnum > 0 AND NOT a.attisdropped \
|
||||
ORDER BY a.attnum",
|
||||
)
|
||||
.bind(schema)
|
||||
.bind(table)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
let full_type = row.get::<Option<String>, _>("full_type").unwrap_or_default();
|
||||
ColumnInfo {
|
||||
name: row.get::<String, _>("column_name"),
|
||||
data_type: full_type,
|
||||
is_nullable: row.get::<bool, _>("is_nullable"),
|
||||
column_default: row.get::<Option<String>, _>("column_default"),
|
||||
is_primary_key: row.get::<bool, _>("is_pk"),
|
||||
extra: None,
|
||||
comment: row.get::<Option<String>, _>("column_comment"),
|
||||
numeric_precision: row.get::<Option<i32>, _>("numeric_precision"),
|
||||
numeric_scale: row.get::<Option<i32>, _>("numeric_scale"),
|
||||
character_maximum_length: row.get::<Option<i32>, _>("character_maximum_length"),
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn execute_query(pool: &PgPool, sql: &str) -> Result<QueryResult, String> {
|
||||
let start = Instant::now();
|
||||
let trimmed = sql.trim().to_uppercase();
|
||||
|
||||
if trimmed.starts_with("SELECT")
|
||||
|| trimmed.starts_with("SHOW")
|
||||
|| trimmed.starts_with("EXPLAIN")
|
||||
|| trimmed.starts_with("WITH")
|
||||
|| trimmed.starts_with("TABLE")
|
||||
{
|
||||
let rows: Vec<PgRow> = sqlx::query(sql)
|
||||
.persistent(false)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let (columns, column_types): (Vec<String>, Vec<String>) = if let Some(first) = rows.first() {
|
||||
let cols = first.columns();
|
||||
(
|
||||
cols.iter().map(|c| c.name().to_string()).collect(),
|
||||
cols.iter().map(|c| c.type_info().name().to_string()).collect(),
|
||||
)
|
||||
} else {
|
||||
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
|
||||
(
|
||||
desc.columns().iter().map(|c| c.name().to_string()).collect(),
|
||||
desc.columns().iter().map(|c| c.type_info().name().to_string()).collect(),
|
||||
)
|
||||
};
|
||||
|
||||
let result_rows: Vec<Vec<serde_json::Value>> = rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
(0..row.len())
|
||||
.map(|i| {
|
||||
pg_value_to_json(
|
||||
row,
|
||||
i,
|
||||
column_types.get(i).map(String::as_str).unwrap_or(""),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(QueryResult {
|
||||
columns,
|
||||
rows: result_rows,
|
||||
affected_rows: 0,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
})
|
||||
} else {
|
||||
let result = sqlx::query(sql)
|
||||
.execute(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(QueryResult {
|
||||
columns: vec![],
|
||||
rows: vec![],
|
||||
affected_rows: result.rows_affected(),
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn list_indexes(pool: &PgPool, schema: &str, table: &str) -> Result<Vec<IndexInfo>, String> {
|
||||
let rows: Vec<PgRow> = sqlx::query(
|
||||
"SELECT i.relname AS index_name, \
|
||||
array_agg(COALESCE(a.attname, pg_get_indexdef(ix.indexrelid, k.n::int, true)) ORDER BY k.n) AS columns, \
|
||||
ix.indisunique AS is_unique, \
|
||||
ix.indisprimary AS is_primary, \
|
||||
pg_get_expr(ix.indpred, ix.indrelid) AS filter_expr, \
|
||||
am.amname AS index_type, \
|
||||
ix.indnkeyatts AS nkeyatts, \
|
||||
ix.indkey AS indkey, \
|
||||
obj_description(i.oid, 'pg_class') AS index_comment \
|
||||
FROM pg_index ix \
|
||||
JOIN pg_class t ON t.oid = ix.indrelid \
|
||||
JOIN pg_class i ON i.oid = ix.indexrelid \
|
||||
JOIN pg_namespace n ON n.oid = t.relnamespace \
|
||||
JOIN pg_am am ON am.oid = i.relam \
|
||||
JOIN LATERAL unnest(ix.indkey) WITH ORDINALITY AS k(attnum, n) ON true \
|
||||
LEFT JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum AND k.attnum > 0 \
|
||||
WHERE n.nspname = $1 AND t.relname = $2 \
|
||||
GROUP BY i.relname, i.oid, ix.indisunique, ix.indisprimary, ix.indpred, ix.indrelid, am.amname, ix.indnkeyatts, ix.indkey \
|
||||
ORDER BY i.relname",
|
||||
)
|
||||
.bind(schema)
|
||||
.bind(table)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
let all_cols: Vec<String> = row.get::<Vec<String>, _>("columns");
|
||||
let nkeyatts = row.get::<Option<i16>, _>("nkeyatts").unwrap_or(all_cols.len() as i16) as usize;
|
||||
let key_cols = all_cols[..nkeyatts].to_vec();
|
||||
let included = if nkeyatts < all_cols.len() { all_cols[nkeyatts..].to_vec() } else { vec![] };
|
||||
IndexInfo {
|
||||
name: row.get::<String, _>("index_name"),
|
||||
columns: key_cols,
|
||||
is_unique: row.get::<bool, _>("is_unique"),
|
||||
is_primary: row.get::<bool, _>("is_primary"),
|
||||
filter: row.get::<Option<String>, _>("filter_expr"),
|
||||
index_type: row.get::<Option<String>, _>("index_type"),
|
||||
included_columns: if included.is_empty() { None } else { Some(included) },
|
||||
comment: row.get::<Option<String>, _>("index_comment"),
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn list_foreign_keys(pool: &PgPool, schema: &str, table: &str) -> Result<Vec<ForeignKeyInfo>, String> {
|
||||
let rows: Vec<PgRow> = sqlx::query(
|
||||
"SELECT kcu.constraint_name, kcu.column_name, \
|
||||
ccu.table_name AS ref_table, ccu.column_name AS ref_column \
|
||||
FROM information_schema.key_column_usage kcu \
|
||||
JOIN information_schema.referential_constraints rc \
|
||||
ON kcu.constraint_name = rc.constraint_name \
|
||||
AND kcu.constraint_schema = rc.constraint_schema \
|
||||
JOIN information_schema.constraint_column_usage ccu \
|
||||
ON rc.unique_constraint_name = ccu.constraint_name \
|
||||
AND rc.unique_constraint_schema = ccu.constraint_schema \
|
||||
WHERE kcu.table_schema = $1 AND kcu.table_name = $2 \
|
||||
ORDER BY kcu.constraint_name",
|
||||
)
|
||||
.bind(schema)
|
||||
.bind(table)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(rows
|
||||
.iter()
|
||||
.map(|row| ForeignKeyInfo {
|
||||
name: row.get::<String, _>("constraint_name"),
|
||||
column: row.get::<String, _>("column_name"),
|
||||
ref_table: row.get::<String, _>("ref_table"),
|
||||
ref_column: row.get::<String, _>("ref_column"),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn list_triggers(pool: &PgPool, schema: &str, table: &str) -> Result<Vec<TriggerInfo>, String> {
|
||||
let rows: Vec<PgRow> = sqlx::query(
|
||||
"SELECT trigger_name, event_manipulation, action_timing \
|
||||
FROM information_schema.triggers \
|
||||
WHERE trigger_schema = $1 AND event_object_table = $2 \
|
||||
ORDER BY trigger_name",
|
||||
)
|
||||
.bind(schema)
|
||||
.bind(table)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(rows
|
||||
.iter()
|
||||
.map(|row| TriggerInfo {
|
||||
name: row.get::<String, _>("trigger_name"),
|
||||
event: row.get::<String, _>("event_manipulation"),
|
||||
timing: row.get::<String, _>("action_timing"),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
@@ -0,0 +1,443 @@
|
||||
use redis::{AsyncCommands, FromRedisValue, Value as RedisRawValue};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
const STREAM_ENTRY_LIMIT: usize = 100;
|
||||
const DEFAULT_REDIS_DATABASES: u32 = 16;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RedisKeyInfo {
|
||||
pub key: String,
|
||||
pub key_type: String,
|
||||
pub ttl: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RedisScanResult {
|
||||
pub cursor: u64,
|
||||
pub keys: Vec<RedisKeyInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RedisValue {
|
||||
pub key: String,
|
||||
pub key_type: String,
|
||||
pub ttl: i64,
|
||||
pub value: serde_json::Value,
|
||||
}
|
||||
|
||||
pub async fn connect(url: &str) -> Result<redis::aio::MultiplexedConnection, String> {
|
||||
let client = redis::Client::open(url).map_err(|e| format!("Redis connection failed: {e}"))?;
|
||||
let mut con = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(10),
|
||||
client.get_multiplexed_async_connection(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| "Redis connection timed out (10s)".to_string())?
|
||||
.map_err(|e| format!("Redis connection failed: {e}"))?;
|
||||
|
||||
redis::cmd("PING")
|
||||
.query_async::<String>(&mut con)
|
||||
.await
|
||||
.map_err(|e| format!("Redis authentication failed or command rejected: {e}"))?;
|
||||
|
||||
Ok(con)
|
||||
}
|
||||
|
||||
pub async fn list_databases(
|
||||
con: &mut redis::aio::MultiplexedConnection,
|
||||
) -> Result<Vec<u32>, String> {
|
||||
let configured_count = redis::cmd("CONFIG")
|
||||
.arg("GET")
|
||||
.arg("databases")
|
||||
.query_async(con)
|
||||
.await
|
||||
.ok()
|
||||
.and_then(parse_database_count);
|
||||
|
||||
let keyspace_dbs = list_keyspace_databases(con).await.unwrap_or_default();
|
||||
let database_count = configured_count.unwrap_or(DEFAULT_REDIS_DATABASES);
|
||||
let max_db = keyspace_dbs
|
||||
.iter()
|
||||
.copied()
|
||||
.max()
|
||||
.map(|db| db + 1)
|
||||
.unwrap_or(0);
|
||||
let visible_count = database_count.max(max_db).max(1);
|
||||
|
||||
Ok((0..visible_count).collect())
|
||||
}
|
||||
|
||||
fn parse_database_count(value: redis::Value) -> Option<u32> {
|
||||
let values = match value {
|
||||
redis::Value::Array(values) => values,
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
values.windows(2).find_map(|pair| {
|
||||
let key = String::from_redis_value(&pair[0]).ok()?;
|
||||
if key.eq_ignore_ascii_case("databases") {
|
||||
String::from_redis_value(&pair[1]).ok()?.parse().ok()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async fn list_keyspace_databases(
|
||||
con: &mut redis::aio::MultiplexedConnection,
|
||||
) -> Result<Vec<u32>, String> {
|
||||
let info: String = redis::cmd("INFO")
|
||||
.arg("keyspace")
|
||||
.query_async(con)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let mut dbs = Vec::new();
|
||||
for line in info.lines() {
|
||||
if line.starts_with("db") {
|
||||
if let Some(num) = line.strip_prefix("db").and_then(|s| s.split(':').next()) {
|
||||
if let Ok(n) = num.parse::<u32>() {
|
||||
dbs.push(n);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(dbs)
|
||||
}
|
||||
|
||||
pub async fn select_db(con: &mut redis::aio::MultiplexedConnection, db: u32) -> Result<(), String> {
|
||||
redis::cmd("SELECT")
|
||||
.arg(db)
|
||||
.query_async(con)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub async fn scan_keys_page(
|
||||
con: &mut redis::aio::MultiplexedConnection,
|
||||
cursor: u64,
|
||||
pattern: &str,
|
||||
count: usize,
|
||||
) -> Result<RedisScanResult, String> {
|
||||
let (next_cursor, keys): (u64, Vec<String>) = redis::cmd("SCAN")
|
||||
.arg(cursor)
|
||||
.arg("MATCH")
|
||||
.arg(pattern)
|
||||
.arg("COUNT")
|
||||
.arg(count)
|
||||
.query_async(con)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let mut result = Vec::new();
|
||||
for key in &keys {
|
||||
let key_type: String = redis::cmd("TYPE")
|
||||
.arg(key.as_str())
|
||||
.query_async(con)
|
||||
.await
|
||||
.unwrap_or_else(|_| "unknown".to_string());
|
||||
|
||||
let ttl: i64 = con.ttl(key.as_str()).await.unwrap_or(-1);
|
||||
|
||||
result.push(RedisKeyInfo {
|
||||
key: key.clone(),
|
||||
key_type,
|
||||
ttl,
|
||||
});
|
||||
}
|
||||
Ok(RedisScanResult {
|
||||
cursor: next_cursor,
|
||||
keys: result,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn get_value(
|
||||
con: &mut redis::aio::MultiplexedConnection,
|
||||
key: &str,
|
||||
) -> Result<RedisValue, String> {
|
||||
let key_type: String = redis::cmd("TYPE")
|
||||
.arg(key)
|
||||
.query_async(con)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let ttl: i64 = con.ttl(key).await.unwrap_or(-1);
|
||||
|
||||
let value = match key_type.as_str() {
|
||||
"string" => {
|
||||
let v: String = con.get(key).await.map_err(|e| e.to_string())?;
|
||||
serde_json::Value::String(v)
|
||||
}
|
||||
"list" => {
|
||||
let v: Vec<String> = con.lrange(key, 0, -1).await.map_err(|e| e.to_string())?;
|
||||
serde_json::json!(v)
|
||||
}
|
||||
"set" => {
|
||||
let v: Vec<String> = con.smembers(key).await.map_err(|e| e.to_string())?;
|
||||
serde_json::json!(v)
|
||||
}
|
||||
"zset" => {
|
||||
let v: Vec<(String, f64)> = con
|
||||
.zrange_withscores(key, 0, -1)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
serde_json::json!(v
|
||||
.iter()
|
||||
.map(|(m, s)| serde_json::json!({"member": m, "score": s}))
|
||||
.collect::<Vec<_>>())
|
||||
}
|
||||
"hash" => {
|
||||
let v: Vec<(String, String)> = con.hgetall(key).await.map_err(|e| e.to_string())?;
|
||||
let map: serde_json::Map<String, serde_json::Value> = v
|
||||
.into_iter()
|
||||
.map(|(k, v)| (k, serde_json::Value::String(v)))
|
||||
.collect();
|
||||
serde_json::Value::Object(map)
|
||||
}
|
||||
"stream" => get_stream_entries(con, key).await?,
|
||||
_ => serde_json::Value::Null,
|
||||
};
|
||||
|
||||
Ok(RedisValue {
|
||||
key: key.to_string(),
|
||||
key_type,
|
||||
ttl,
|
||||
value,
|
||||
})
|
||||
}
|
||||
|
||||
async fn get_stream_entries(
|
||||
con: &mut redis::aio::MultiplexedConnection,
|
||||
key: &str,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
let raw: RedisRawValue = redis::cmd("XRANGE")
|
||||
.arg(key)
|
||||
.arg("-")
|
||||
.arg("+")
|
||||
.arg("COUNT")
|
||||
.arg(STREAM_ENTRY_LIMIT)
|
||||
.query_async(con)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(parse_stream_entries(raw))
|
||||
}
|
||||
|
||||
fn parse_stream_entries(raw: RedisRawValue) -> serde_json::Value {
|
||||
match raw {
|
||||
RedisRawValue::Array(entries) => {
|
||||
serde_json::Value::Array(entries.into_iter().filter_map(parse_stream_entry).collect())
|
||||
}
|
||||
_ => serde_json::Value::Null,
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_stream_entry(entry: RedisRawValue) -> Option<serde_json::Value> {
|
||||
let mut parts = match entry {
|
||||
RedisRawValue::Array(parts) if parts.len() == 2 => parts.into_iter(),
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let id = redis_value_to_string(parts.next()?)?;
|
||||
let fields = match parts.next()? {
|
||||
RedisRawValue::Array(fields) => fields,
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let mut field_map = serde_json::Map::new();
|
||||
let mut fields = fields.into_iter();
|
||||
while let Some(field) = fields.next() {
|
||||
let Some(value) = fields.next() else {
|
||||
break;
|
||||
};
|
||||
if let Some(field_name) = redis_value_to_string(field) {
|
||||
let value = redis_value_to_string(value).unwrap_or_default();
|
||||
field_map.insert(field_name, serde_json::Value::String(value));
|
||||
}
|
||||
}
|
||||
|
||||
Some(serde_json::json!({
|
||||
"id": id,
|
||||
"fields": field_map,
|
||||
}))
|
||||
}
|
||||
|
||||
fn redis_value_to_string(value: RedisRawValue) -> Option<String> {
|
||||
match value {
|
||||
RedisRawValue::BulkString(bytes) => Some(String::from_utf8_lossy(&bytes).to_string()),
|
||||
RedisRawValue::SimpleString(value) => Some(value),
|
||||
RedisRawValue::Int(value) => Some(value.to_string()),
|
||||
RedisRawValue::Double(value) => Some(value.to_string()),
|
||||
RedisRawValue::Boolean(value) => Some(value.to_string()),
|
||||
RedisRawValue::VerbatimString { text, .. } => Some(text),
|
||||
RedisRawValue::Okay => Some("OK".to_string()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn set_string(
|
||||
con: &mut redis::aio::MultiplexedConnection,
|
||||
key: &str,
|
||||
value: &str,
|
||||
ttl: Option<i64>,
|
||||
) -> Result<(), String> {
|
||||
con.set::<_, _, ()>(key, value)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
if let Some(t) = ttl {
|
||||
if t > 0 {
|
||||
con.expire::<_, ()>(key, t)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn delete_key(
|
||||
con: &mut redis::aio::MultiplexedConnection,
|
||||
key: &str,
|
||||
) -> Result<(), String> {
|
||||
con.del::<_, ()>(key).await.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub async fn hash_set(
|
||||
con: &mut redis::aio::MultiplexedConnection,
|
||||
key: &str,
|
||||
field: &str,
|
||||
value: &str,
|
||||
) -> Result<(), String> {
|
||||
con.hset::<_, _, _, ()>(key, field, value)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub async fn hash_del(
|
||||
con: &mut redis::aio::MultiplexedConnection,
|
||||
key: &str,
|
||||
field: &str,
|
||||
) -> Result<(), String> {
|
||||
con.hdel::<_, _, ()>(key, field)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub async fn list_push(
|
||||
con: &mut redis::aio::MultiplexedConnection,
|
||||
key: &str,
|
||||
value: &str,
|
||||
) -> Result<(), String> {
|
||||
con.rpush::<_, _, ()>(key, value)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub async fn list_remove(
|
||||
con: &mut redis::aio::MultiplexedConnection,
|
||||
key: &str,
|
||||
index: i64,
|
||||
) -> Result<(), String> {
|
||||
let placeholder = "__DELETED_PLACEHOLDER__";
|
||||
redis::cmd("LSET")
|
||||
.arg(key)
|
||||
.arg(index)
|
||||
.arg(placeholder)
|
||||
.query_async::<()>(con)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
con.lrem::<_, _, ()>(key, 1, placeholder)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub async fn set_add(
|
||||
con: &mut redis::aio::MultiplexedConnection,
|
||||
key: &str,
|
||||
member: &str,
|
||||
) -> Result<(), String> {
|
||||
con.sadd::<_, _, ()>(key, member)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub async fn set_remove(
|
||||
con: &mut redis::aio::MultiplexedConnection,
|
||||
key: &str,
|
||||
member: &str,
|
||||
) -> Result<(), String> {
|
||||
con.srem::<_, _, ()>(key, member)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{parse_database_count, parse_stream_entries, RedisRawValue};
|
||||
|
||||
fn bulk(value: &str) -> RedisRawValue {
|
||||
RedisRawValue::BulkString(value.as_bytes().to_vec())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_stream_entries() {
|
||||
let raw = RedisRawValue::Array(vec![RedisRawValue::Array(vec![
|
||||
bulk("1714470000000-0"),
|
||||
RedisRawValue::Array(vec![
|
||||
bulk("event"),
|
||||
bulk("login"),
|
||||
bulk("user_id"),
|
||||
bulk("42"),
|
||||
]),
|
||||
])]);
|
||||
|
||||
let parsed = parse_stream_entries(raw);
|
||||
|
||||
assert_eq!(
|
||||
parsed,
|
||||
serde_json::json!([
|
||||
{
|
||||
"id": "1714470000000-0",
|
||||
"fields": {
|
||||
"event": "login",
|
||||
"user_id": "42"
|
||||
}
|
||||
}
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skips_malformed_stream_entries() {
|
||||
let raw = RedisRawValue::Array(vec![
|
||||
RedisRawValue::Array(vec![bulk("1714470000000-0")]),
|
||||
RedisRawValue::Array(vec![
|
||||
bulk("1714470000001-0"),
|
||||
RedisRawValue::Array(vec![bulk("event"), bulk("logout")]),
|
||||
]),
|
||||
]);
|
||||
|
||||
let parsed = parse_stream_entries(raw);
|
||||
|
||||
assert_eq!(
|
||||
parsed,
|
||||
serde_json::json!([
|
||||
{
|
||||
"id": "1714470000001-0",
|
||||
"fields": {
|
||||
"event": "logout"
|
||||
}
|
||||
}
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_configured_database_count() {
|
||||
let value = RedisRawValue::Array(vec![
|
||||
RedisRawValue::BulkString(b"databases".to_vec()),
|
||||
RedisRawValue::BulkString(b"32".to_vec()),
|
||||
]);
|
||||
|
||||
assert_eq!(parse_database_count(value), Some(32));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
use sqlx::sqlite::{SqliteConnectOptions, SqlitePool, SqlitePoolOptions, SqliteRow};
|
||||
use sqlx::{Column, Executor, Row};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::types::{ColumnInfo, DatabaseInfo, ForeignKeyInfo, IndexInfo, QueryResult, TableInfo, TriggerInfo};
|
||||
|
||||
pub async fn connect_path(path: &str) -> Result<SqlitePool, String> {
|
||||
let mut options = SqliteConnectOptions::new()
|
||||
.filename(path)
|
||||
.create_if_missing(true);
|
||||
|
||||
if is_network_path(path) {
|
||||
options = options.vfs("unix-nolock");
|
||||
}
|
||||
|
||||
SqlitePoolOptions::new()
|
||||
.max_connections(5)
|
||||
.acquire_timeout(Duration::from_secs(10))
|
||||
.idle_timeout(Duration::from_secs(300))
|
||||
.connect_with(options)
|
||||
.await
|
||||
.map_err(|e| format!("SQLite connection failed: {e}"))
|
||||
}
|
||||
|
||||
fn is_network_path(path: &str) -> bool {
|
||||
path.starts_with("\\\\") || path.starts_with("//") || path.contains("wsl.localhost") || path.contains("wsl$")
|
||||
}
|
||||
|
||||
pub async fn list_databases(_pool: &SqlitePool) -> Result<Vec<DatabaseInfo>, String> {
|
||||
Ok(vec![DatabaseInfo { name: "main".to_string() }])
|
||||
}
|
||||
|
||||
pub async fn list_tables(pool: &SqlitePool, _schema: &str) -> Result<Vec<TableInfo>, String> {
|
||||
let rows: Vec<SqliteRow> = sqlx::query(
|
||||
"SELECT name, type FROM sqlite_master WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite_%' ORDER BY name",
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
let t: String = row.get("type");
|
||||
TableInfo {
|
||||
name: row.get::<String, _>("name"),
|
||||
table_type: if t == "view" { "VIEW".to_string() } else { "BASE TABLE".to_string() },
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn get_columns(pool: &SqlitePool, _schema: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
|
||||
let rows: Vec<SqliteRow> = sqlx::query(&format!("PRAGMA table_info(\"{}\")", table))
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(rows
|
||||
.iter()
|
||||
.map(|row| ColumnInfo {
|
||||
name: row.get::<String, _>("name"),
|
||||
data_type: row.get::<String, _>("type"),
|
||||
is_nullable: row.get::<i32, _>("notnull") == 0,
|
||||
column_default: row.get::<Option<String>, _>("dflt_value"),
|
||||
is_primary_key: row.get::<i32, _>("pk") > 0,
|
||||
extra: None, comment: None,
|
||||
numeric_precision: None,
|
||||
numeric_scale: None,
|
||||
character_maximum_length: None,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn list_indexes(pool: &SqlitePool, _schema: &str, table: &str) -> Result<Vec<IndexInfo>, String> {
|
||||
let safe_table = table.replace('"', "\"\"");
|
||||
let idx_rows: Vec<SqliteRow> = sqlx::query(&format!("PRAGMA index_list(\"{safe_table}\")"))
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let mut indexes = Vec::new();
|
||||
for idx_row in &idx_rows {
|
||||
let name: String = idx_row.get("name");
|
||||
let is_unique: bool = idx_row.get::<i32, _>("unique") != 0;
|
||||
let origin: String = idx_row.get::<String, _>("origin");
|
||||
let is_primary = origin == "pk";
|
||||
|
||||
let safe_name = name.replace('"', "\"\"");
|
||||
let col_rows: Vec<SqliteRow> = sqlx::query(&format!("PRAGMA index_info(\"{safe_name}\")"))
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let columns: Vec<String> = col_rows.iter().map(|r| r.get::<String, _>("name")).collect();
|
||||
|
||||
indexes.push(IndexInfo {
|
||||
name,
|
||||
columns,
|
||||
is_unique,
|
||||
is_primary,
|
||||
filter: None,
|
||||
index_type: None,
|
||||
included_columns: None,
|
||||
comment: None,
|
||||
});
|
||||
}
|
||||
Ok(indexes)
|
||||
}
|
||||
|
||||
pub async fn list_foreign_keys(pool: &SqlitePool, _schema: &str, table: &str) -> Result<Vec<ForeignKeyInfo>, String> {
|
||||
let rows: Vec<SqliteRow> = sqlx::query(&format!("PRAGMA foreign_key_list(\"{}\")", table))
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(rows
|
||||
.iter()
|
||||
.map(|row| ForeignKeyInfo {
|
||||
name: format!("fk_{}", row.get::<i32, _>("id")),
|
||||
column: row.get::<String, _>("from"),
|
||||
ref_table: row.get::<String, _>("table"),
|
||||
ref_column: row.get::<String, _>("to"),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn list_triggers(pool: &SqlitePool, _schema: &str, table: &str) -> Result<Vec<TriggerInfo>, String> {
|
||||
let rows: Vec<SqliteRow> = sqlx::query(
|
||||
"SELECT name, sql FROM sqlite_master WHERE type = 'trigger' AND tbl_name = ? ORDER BY name",
|
||||
)
|
||||
.bind(table)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
let sql_text: String = row.get::<Option<String>, _>("sql").unwrap_or_default();
|
||||
let upper = sql_text.to_uppercase();
|
||||
let timing = if upper.contains("BEFORE") { "BEFORE" } else if upper.contains("AFTER") { "AFTER" } else { "INSTEAD OF" };
|
||||
let event = if upper.contains("INSERT") { "INSERT" } else if upper.contains("UPDATE") { "UPDATE" } else { "DELETE" };
|
||||
TriggerInfo {
|
||||
name: row.get::<String, _>("name"),
|
||||
event: event.to_string(),
|
||||
timing: timing.to_string(),
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn execute_query(pool: &SqlitePool, sql: &str) -> Result<QueryResult, String> {
|
||||
let start = Instant::now();
|
||||
let trimmed = sql.trim().to_uppercase();
|
||||
|
||||
if trimmed.starts_with("SELECT")
|
||||
|| trimmed.starts_with("PRAGMA")
|
||||
|| trimmed.starts_with("EXPLAIN")
|
||||
|| trimmed.starts_with("WITH")
|
||||
{
|
||||
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
|
||||
let columns: Vec<String> = desc.columns().iter().map(|c| c.name().to_string()).collect();
|
||||
|
||||
let rows: Vec<SqliteRow> = sqlx::query(sql)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let result_rows: Vec<Vec<serde_json::Value>> = rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
(0..row.len())
|
||||
.map(|i| {
|
||||
row.try_get::<String, _>(i)
|
||||
.map(serde_json::Value::String)
|
||||
.or_else(|_| row.try_get::<i64, _>(i).map(|v| serde_json::Value::Number(v.into())))
|
||||
.or_else(|_| row.try_get::<f64, _>(i).map(|v| {
|
||||
serde_json::Number::from_f64(v)
|
||||
.map(serde_json::Value::Number)
|
||||
.unwrap_or(serde_json::Value::Null)
|
||||
}))
|
||||
.or_else(|_| row.try_get::<bool, _>(i).map(serde_json::Value::Bool))
|
||||
.unwrap_or(serde_json::Value::Null)
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(QueryResult {
|
||||
columns,
|
||||
rows: result_rows,
|
||||
affected_rows: 0,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
})
|
||||
} else {
|
||||
let result = sqlx::query(sql)
|
||||
.execute(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(QueryResult {
|
||||
columns: vec![],
|
||||
rows: vec![],
|
||||
affected_rows: result.rows_affected(),
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
use rust_decimal::Decimal;
|
||||
use tiberius::{AuthMethod, Client, Config};
|
||||
use tokio::net::TcpStream;
|
||||
use tokio_util::compat::{Compat, TokioAsyncWriteCompatExt};
|
||||
use std::time::Instant;
|
||||
|
||||
use crate::types::{ColumnInfo, DatabaseInfo, ForeignKeyInfo, IndexInfo, QueryResult, TableInfo, TriggerInfo};
|
||||
|
||||
pub type SqlServerClient = Client<Compat<TcpStream>>;
|
||||
|
||||
pub async fn connect(host: &str, port: u16, user: &str, pass: &str, database: Option<&str>) -> Result<SqlServerClient, String> {
|
||||
match try_connect(host, port, user, pass, database, true).await {
|
||||
Ok(client) => Ok(client),
|
||||
Err(_) => try_connect(host, port, user, pass, database, false).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn try_connect(host: &str, port: u16, user: &str, pass: &str, database: Option<&str>, use_encryption: bool) -> Result<SqlServerClient, String> {
|
||||
let mut config = Config::new();
|
||||
config.host(host);
|
||||
config.port(port);
|
||||
config.authentication(AuthMethod::sql_server(user, pass));
|
||||
if let Some(db) = database {
|
||||
config.database(db);
|
||||
}
|
||||
config.trust_cert();
|
||||
if !use_encryption {
|
||||
config.encryption(tiberius::EncryptionLevel::NotSupported);
|
||||
}
|
||||
|
||||
let tcp = TcpStream::connect(config.get_addr())
|
||||
.await
|
||||
.map_err(|e| format!("SQL Server connection failed: {e}"))?;
|
||||
Client::connect(config, tcp.compat_write())
|
||||
.await
|
||||
.map_err(|e| format!("SQL Server connection failed: {e}"))
|
||||
}
|
||||
|
||||
fn row_to_json(row: &tiberius::Row) -> Vec<serde_json::Value> {
|
||||
(0..row.len()).map(|i| {
|
||||
if let Some(v) = row.try_get::<&str, _>(i).ok().flatten() {
|
||||
serde_json::Value::String(v.to_string())
|
||||
} else if let Some(v) = row.try_get::<Decimal, _>(i).ok().flatten() {
|
||||
serde_json::Value::String(v.to_string())
|
||||
} else if let Some(v) = row.try_get::<i32, _>(i).ok().flatten() {
|
||||
serde_json::Value::Number(v.into())
|
||||
} else if let Some(v) = row.try_get::<i64, _>(i).ok().flatten() {
|
||||
serde_json::Value::Number(v.into())
|
||||
} else if let Some(v) = row.try_get::<f64, _>(i).ok().flatten() {
|
||||
serde_json::Number::from_f64(v).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null)
|
||||
} else if let Some(v) = row.try_get::<bool, _>(i).ok().flatten() {
|
||||
serde_json::Value::Bool(v)
|
||||
} else {
|
||||
serde_json::Value::Null
|
||||
}
|
||||
}).collect()
|
||||
}
|
||||
|
||||
pub async fn list_databases(client: &mut SqlServerClient) -> Result<Vec<DatabaseInfo>, String> {
|
||||
let stream = client.query("SELECT name FROM sys.databases ORDER BY name", &[])
|
||||
.await.map_err(|e| e.to_string())?;
|
||||
let rows = stream.into_first_result().await.map_err(|e| e.to_string())?;
|
||||
Ok(rows.iter().map(|row| {
|
||||
DatabaseInfo { name: row.get::<&str, _>(0).unwrap_or("").to_string() }
|
||||
}).collect())
|
||||
}
|
||||
|
||||
pub async fn list_schemas(client: &mut SqlServerClient) -> Result<Vec<String>, String> {
|
||||
let stream = client.query(
|
||||
"SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA \
|
||||
WHERE SCHEMA_NAME NOT IN ('guest','INFORMATION_SCHEMA','sys') \
|
||||
ORDER BY SCHEMA_NAME",
|
||||
&[],
|
||||
).await.map_err(|e| e.to_string())?;
|
||||
let rows = stream.into_first_result().await.map_err(|e| e.to_string())?;
|
||||
Ok(rows.iter().map(|row| {
|
||||
row.get::<&str, _>(0).unwrap_or("").to_string()
|
||||
}).collect())
|
||||
}
|
||||
|
||||
pub async fn list_tables(client: &mut SqlServerClient, schema: &str) -> Result<Vec<TableInfo>, String> {
|
||||
let sql = format!(
|
||||
"SELECT TABLE_NAME, TABLE_TYPE FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{}' ORDER BY TABLE_NAME",
|
||||
schema.replace('\'', "''")
|
||||
);
|
||||
let stream = client.query(&*sql, &[]).await.map_err(|e| e.to_string())?;
|
||||
let rows = stream.into_first_result().await.map_err(|e| e.to_string())?;
|
||||
Ok(rows.iter().map(|row| {
|
||||
TableInfo {
|
||||
name: row.get::<&str, _>(0).unwrap_or("").to_string(),
|
||||
table_type: row.get::<&str, _>(1).unwrap_or("BASE TABLE").to_string(),
|
||||
}
|
||||
}).collect())
|
||||
}
|
||||
|
||||
pub async fn get_columns(client: &mut SqlServerClient, schema: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
|
||||
let sql = format!(
|
||||
"SELECT c.COLUMN_NAME, c.DATA_TYPE, c.IS_NULLABLE, c.COLUMN_DEFAULT, \
|
||||
CASE WHEN kcu.COLUMN_NAME IS NOT NULL THEN 1 ELSE 0 END AS IS_PK, \
|
||||
c.NUMERIC_PRECISION, c.NUMERIC_SCALE, c.CHARACTER_MAXIMUM_LENGTH, c.DATETIME_PRECISION \
|
||||
FROM INFORMATION_SCHEMA.COLUMNS c \
|
||||
LEFT JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu \
|
||||
ON c.TABLE_SCHEMA = kcu.TABLE_SCHEMA AND c.TABLE_NAME = kcu.TABLE_NAME AND c.COLUMN_NAME = kcu.COLUMN_NAME \
|
||||
AND kcu.CONSTRAINT_NAME IN (SELECT CONSTRAINT_NAME FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE CONSTRAINT_TYPE = 'PRIMARY KEY' AND TABLE_SCHEMA = '{s}' AND TABLE_NAME = '{t}') \
|
||||
WHERE c.TABLE_SCHEMA = '{s}' AND c.TABLE_NAME = '{t}' \
|
||||
ORDER BY c.ORDINAL_POSITION",
|
||||
s = schema.replace('\'', "''"), t = table.replace('\'', "''")
|
||||
);
|
||||
let stream = client.query(&*sql, &[]).await.map_err(|e| e.to_string())?;
|
||||
let rows = stream.into_first_result().await.map_err(|e| e.to_string())?;
|
||||
Ok(rows.iter().map(|row| {
|
||||
let base = row.get::<&str, _>(1).unwrap_or("").to_string();
|
||||
let max_len = row.get::<i32, _>(7);
|
||||
let dt_prec = row.get::<i32, _>(8);
|
||||
let num_prec = row.get::<i32, _>(5);
|
||||
let num_scale = row.get::<i32, _>(6);
|
||||
let data_type = match base.to_lowercase().as_str() {
|
||||
"varchar" => match max_len {
|
||||
Some(-1) => "varchar(max)".to_string(),
|
||||
Some(n) => format!("varchar({n})"),
|
||||
None => "varchar".to_string(),
|
||||
},
|
||||
"nvarchar" => match max_len {
|
||||
Some(-1) => "nvarchar(max)".to_string(),
|
||||
Some(n) => format!("nvarchar({n})"),
|
||||
None => "nvarchar".to_string(),
|
||||
},
|
||||
"varbinary" => match max_len {
|
||||
Some(-1) => "varbinary(max)".to_string(),
|
||||
Some(n) if n > 0 => format!("varbinary({n})"),
|
||||
_ => "varbinary".to_string(),
|
||||
},
|
||||
"char" | "nchar" | "binary" => match max_len {
|
||||
Some(n) if n > 0 => format!("{base}({n})"),
|
||||
_ => base,
|
||||
}
|
||||
"decimal" | "numeric" => match (num_prec, num_scale) {
|
||||
(Some(p), Some(s)) => format!("{base}({p},{s})"),
|
||||
_ => base,
|
||||
},
|
||||
"datetime2" | "datetimeoffset" | "time" => match dt_prec {
|
||||
Some(p) => format!("{base}({p})"),
|
||||
_ => base,
|
||||
},
|
||||
_ => base,
|
||||
};
|
||||
ColumnInfo {
|
||||
name: row.get::<&str, _>(0).unwrap_or("").to_string(),
|
||||
data_type,
|
||||
is_nullable: row.get::<&str, _>(2).unwrap_or("NO") == "YES",
|
||||
column_default: row.get::<&str, _>(3).map(|s| s.to_string()),
|
||||
is_primary_key: row.get::<i32, _>(4).unwrap_or(0) == 1,
|
||||
extra: None, comment: None,
|
||||
numeric_precision: num_prec,
|
||||
numeric_scale: num_scale,
|
||||
character_maximum_length: max_len,
|
||||
}
|
||||
}).collect())
|
||||
}
|
||||
|
||||
pub async fn list_indexes(client: &mut SqlServerClient, schema: &str, table: &str) -> Result<Vec<IndexInfo>, String> {
|
||||
let sql = format!(
|
||||
"SELECT i.name, \
|
||||
STRING_AGG(CASE WHEN ic.is_included_column = 0 THEN c.name END, ',') WITHIN GROUP (ORDER BY ic.key_ordinal) AS columns, \
|
||||
i.is_unique, i.is_primary_key, i.type_desc, \
|
||||
STRING_AGG(CASE WHEN ic.is_included_column = 1 THEN c.name END, ',') AS included_cols, \
|
||||
i.filter_definition \
|
||||
FROM sys.indexes i \
|
||||
JOIN sys.index_columns ic ON i.object_id = ic.object_id AND i.index_id = ic.index_id \
|
||||
JOIN sys.columns c ON ic.object_id = c.object_id AND ic.column_id = c.column_id \
|
||||
WHERE i.object_id = OBJECT_ID('{s}.{t}') AND i.name IS NOT NULL \
|
||||
GROUP BY i.name, i.is_unique, i.is_primary_key, i.type_desc, i.filter_definition \
|
||||
ORDER BY i.name",
|
||||
s = schema.replace('\'', "''"), t = table.replace('\'', "''")
|
||||
);
|
||||
let stream = client.query(&*sql, &[]).await.map_err(|e| e.to_string())?;
|
||||
let rows = stream.into_first_result().await.map_err(|e| e.to_string())?;
|
||||
Ok(rows.iter().map(|row| {
|
||||
let cols_str = row.get::<&str, _>(1).unwrap_or("");
|
||||
let inc_str = row.get::<&str, _>(5).unwrap_or("");
|
||||
IndexInfo {
|
||||
name: row.get::<&str, _>(0).unwrap_or("").to_string(),
|
||||
columns: cols_str.split(',').filter(|s| !s.is_empty()).map(|s| s.to_string()).collect(),
|
||||
is_unique: row.get::<bool, _>(2).unwrap_or(false),
|
||||
is_primary: row.get::<bool, _>(3).unwrap_or(false),
|
||||
filter: row.get::<&str, _>(6).map(|s| s.to_string()),
|
||||
index_type: row.get::<&str, _>(4).map(|s| s.to_string()),
|
||||
included_columns: if inc_str.is_empty() { None } else { Some(inc_str.split(',').map(|s| s.to_string()).collect()) },
|
||||
comment: None,
|
||||
}
|
||||
}).collect())
|
||||
}
|
||||
|
||||
pub async fn list_foreign_keys(client: &mut SqlServerClient, schema: &str, table: &str) -> Result<Vec<ForeignKeyInfo>, String> {
|
||||
let sql = format!(
|
||||
"SELECT fk.name, c.name, rt.name, rc.name \
|
||||
FROM sys.foreign_keys fk \
|
||||
JOIN sys.foreign_key_columns fkc ON fk.object_id = fkc.constraint_object_id \
|
||||
JOIN sys.columns c ON fkc.parent_object_id = c.object_id AND fkc.parent_column_id = c.column_id \
|
||||
JOIN sys.tables rt ON fkc.referenced_object_id = rt.object_id \
|
||||
JOIN sys.columns rc ON fkc.referenced_object_id = rc.object_id AND fkc.referenced_column_id = rc.column_id \
|
||||
WHERE fk.parent_object_id = OBJECT_ID('{s}.{t}') \
|
||||
ORDER BY fk.name",
|
||||
s = schema.replace('\'', "''"), t = table.replace('\'', "''")
|
||||
);
|
||||
let stream = client.query(&*sql, &[]).await.map_err(|e| e.to_string())?;
|
||||
let rows = stream.into_first_result().await.map_err(|e| e.to_string())?;
|
||||
Ok(rows.iter().map(|row| {
|
||||
ForeignKeyInfo {
|
||||
name: row.get::<&str, _>(0).unwrap_or("").to_string(),
|
||||
column: row.get::<&str, _>(1).unwrap_or("").to_string(),
|
||||
ref_table: row.get::<&str, _>(2).unwrap_or("").to_string(),
|
||||
ref_column: row.get::<&str, _>(3).unwrap_or("").to_string(),
|
||||
}
|
||||
}).collect())
|
||||
}
|
||||
|
||||
pub async fn list_triggers(client: &mut SqlServerClient, schema: &str, table: &str) -> Result<Vec<TriggerInfo>, String> {
|
||||
let sql = format!(
|
||||
"SELECT t.name, te.type_desc, CASE WHEN t.is_instead_of_trigger = 1 THEN 'INSTEAD OF' ELSE 'AFTER' END \
|
||||
FROM sys.triggers t \
|
||||
JOIN sys.trigger_events te ON t.object_id = te.object_id \
|
||||
WHERE t.parent_id = OBJECT_ID('{s}.{t}') \
|
||||
ORDER BY t.name",
|
||||
s = schema.replace('\'', "''"), t = table.replace('\'', "''")
|
||||
);
|
||||
let stream = client.query(&*sql, &[]).await.map_err(|e| e.to_string())?;
|
||||
let rows = stream.into_first_result().await.map_err(|e| e.to_string())?;
|
||||
Ok(rows.iter().map(|row| {
|
||||
TriggerInfo {
|
||||
name: row.get::<&str, _>(0).unwrap_or("").to_string(),
|
||||
event: row.get::<&str, _>(1).unwrap_or("").to_string(),
|
||||
timing: row.get::<&str, _>(2).unwrap_or("AFTER").to_string(),
|
||||
}
|
||||
}).collect())
|
||||
}
|
||||
|
||||
pub async fn execute_query(client: &mut SqlServerClient, sql: &str) -> Result<QueryResult, String> {
|
||||
let start = Instant::now();
|
||||
let trimmed = sql.trim().to_uppercase();
|
||||
|
||||
if trimmed.starts_with("SELECT")
|
||||
|| trimmed.starts_with("EXEC")
|
||||
|| trimmed.starts_with("WITH")
|
||||
|| trimmed.starts_with("TABLE")
|
||||
{
|
||||
let mut stream = client.query(sql, &[]).await.map_err(|e| e.to_string())?;
|
||||
let columns_meta = stream.columns().await.map_err(|e| e.to_string())?
|
||||
.map(|cols| cols.iter().map(|c| c.name().to_string()).collect::<Vec<_>>())
|
||||
.unwrap_or_default();
|
||||
|
||||
let rows = stream.into_first_result().await.map_err(|e| e.to_string())?;
|
||||
let result_rows: Vec<Vec<serde_json::Value>> = rows.iter().map(|row| row_to_json(row)).collect();
|
||||
|
||||
Ok(QueryResult {
|
||||
columns: columns_meta,
|
||||
rows: result_rows,
|
||||
affected_rows: 0,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
})
|
||||
} else {
|
||||
let result = client.execute(sql, &[]).await.map_err(|e| e.to_string())?;
|
||||
Ok(QueryResult {
|
||||
columns: vec![],
|
||||
rows: vec![],
|
||||
affected_rows: result.rows_affected().iter().sum::<u64>(),
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use russh::client::{self, Config, Handle};
|
||||
use russh::keys::{key::PrivateKeyWithHashAlg, load_secret_key};
|
||||
use russh::ChannelMsg;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
struct SshClient;
|
||||
|
||||
impl client::Handler for SshClient {
|
||||
type Error = russh::Error;
|
||||
|
||||
async fn check_server_key(
|
||||
&mut self,
|
||||
_server_public_key: &russh::keys::ssh_key::PublicKey,
|
||||
) -> Result<bool, Self::Error> {
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
async fn connect_and_authenticate(
|
||||
ssh_host: &str,
|
||||
ssh_port: u16,
|
||||
ssh_user: &str,
|
||||
ssh_password: &str,
|
||||
ssh_key_path: &str,
|
||||
ssh_key_passphrase: &str,
|
||||
) -> Result<Handle<SshClient>, String> {
|
||||
let config = Arc::new(Config {
|
||||
nodelay: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let mut session = client::connect(config, (ssh_host, ssh_port), SshClient {})
|
||||
.await
|
||||
.map_err(|e| format!("SSH connection failed: {e}"))?;
|
||||
|
||||
if !ssh_key_path.is_empty() {
|
||||
let passphrase = if ssh_key_passphrase.is_empty() { None } else { Some(ssh_key_passphrase) };
|
||||
let key_pair = load_secret_key(ssh_key_path, passphrase)
|
||||
.map_err(|e| format!("Failed to load SSH key: {e}"))?;
|
||||
let auth_res = session
|
||||
.authenticate_publickey(
|
||||
ssh_user,
|
||||
PrivateKeyWithHashAlg::new(
|
||||
Arc::new(key_pair),
|
||||
session
|
||||
.best_supported_rsa_hash()
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.flatten(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| format!("SSH key auth failed: {e}"))?;
|
||||
if !auth_res.success() {
|
||||
return Err("SSH public key authentication failed".to_string());
|
||||
}
|
||||
} else if !ssh_password.is_empty() {
|
||||
let auth_res = session
|
||||
.authenticate_password(ssh_user, ssh_password)
|
||||
.await
|
||||
.map_err(|e| format!("SSH password auth failed: {e}"))?;
|
||||
if !auth_res.success() {
|
||||
return Err("SSH password authentication failed".to_string());
|
||||
}
|
||||
} else {
|
||||
return Err("No SSH password or key provided".to_string());
|
||||
}
|
||||
|
||||
Ok(session)
|
||||
}
|
||||
|
||||
async fn forward_loop(
|
||||
session: Handle<SshClient>,
|
||||
listener: TcpListener,
|
||||
remote_host: String,
|
||||
remote_port: u16,
|
||||
) {
|
||||
loop {
|
||||
let (mut stream, peer_addr) = match listener.accept().await {
|
||||
Ok(v) => v,
|
||||
Err(_) => break,
|
||||
};
|
||||
|
||||
let mut channel = match session
|
||||
.channel_open_direct_tcpip(
|
||||
&remote_host,
|
||||
remote_port.into(),
|
||||
peer_addr.ip().to_string(),
|
||||
peer_addr.port().into(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
log::error!("SSH direct-tcpip failed: {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut buf = vec![0u8; 65536];
|
||||
let mut stream_closed = false;
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
r = stream.read(&mut buf), if !stream_closed => {
|
||||
match r {
|
||||
Ok(0) => {
|
||||
stream_closed = true;
|
||||
let _ = channel.eof().await;
|
||||
}
|
||||
Ok(n) => {
|
||||
if channel.data(&buf[..n]).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
msg = channel.wait() => {
|
||||
match msg {
|
||||
Some(ChannelMsg::Data { ref data }) => {
|
||||
if stream.write_all(data).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Some(ChannelMsg::Eof) | None => break,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TunnelManager {
|
||||
tunnels: Mutex<HashMap<String, (JoinHandle<()>, u16)>>,
|
||||
}
|
||||
|
||||
impl TunnelManager {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
tunnels: Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn start_tunnel(
|
||||
&self,
|
||||
connection_id: &str,
|
||||
ssh_host: &str,
|
||||
ssh_port: u16,
|
||||
ssh_user: &str,
|
||||
ssh_password: &str,
|
||||
ssh_key_path: &str,
|
||||
ssh_key_passphrase: &str,
|
||||
remote_host: &str,
|
||||
remote_port: u16,
|
||||
expose_to_lan: bool,
|
||||
) -> Result<u16, String> {
|
||||
let local_port = portpicker::pick_unused_port().ok_or("No available port")?;
|
||||
|
||||
let session =
|
||||
connect_and_authenticate(ssh_host, ssh_port, ssh_user, ssh_password, ssh_key_path, ssh_key_passphrase)
|
||||
.await?;
|
||||
|
||||
let bind_addr = if expose_to_lan { "0.0.0.0" } else { "127.0.0.1" };
|
||||
let listener = TcpListener::bind((bind_addr, local_port))
|
||||
.await
|
||||
.map_err(|e| format!("Failed to bind local port: {e}"))?;
|
||||
|
||||
let remote_host = remote_host.to_string();
|
||||
let handle = tokio::spawn(forward_loop(session, listener, remote_host, remote_port));
|
||||
|
||||
self.tunnels
|
||||
.lock()
|
||||
.await
|
||||
.insert(connection_id.to_string(), (handle, local_port));
|
||||
|
||||
Ok(local_port)
|
||||
}
|
||||
|
||||
pub async fn local_port(&self, connection_id: &str) -> Option<u16> {
|
||||
self.tunnels
|
||||
.lock()
|
||||
.await
|
||||
.get(connection_id)
|
||||
.map(|(_, port)| *port)
|
||||
}
|
||||
|
||||
pub async fn stop_tunnel(&self, connection_id: &str) {
|
||||
if let Some((handle, _)) = self.tunnels.lock().await.remove(connection_id) {
|
||||
handle.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::Path;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct HistoryEntry {
|
||||
pub id: String,
|
||||
pub connection_name: String,
|
||||
pub database: String,
|
||||
pub sql: String,
|
||||
pub executed_at: String,
|
||||
pub execution_time_ms: u128,
|
||||
pub success: bool,
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
pub const MAX_HISTORY: usize = 1000;
|
||||
|
||||
pub fn read_all(path: &Path) -> Result<Vec<HistoryEntry>, String> {
|
||||
if !path.exists() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
let json = std::fs::read_to_string(path).map_err(|e| e.to_string())?;
|
||||
serde_json::from_str(&json).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub fn write_all(path: &Path, entries: &[HistoryEntry]) -> Result<(), String> {
|
||||
let json = serde_json::to_string(entries).map_err(|e| e.to_string())?;
|
||||
std::fs::write(path, json).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub fn save_history_entry(path: &Path, entry: HistoryEntry) -> Result<(), String> {
|
||||
let mut entries = read_all(path)?;
|
||||
entries.insert(0, entry);
|
||||
entries.truncate(MAX_HISTORY);
|
||||
write_all(path, &entries)
|
||||
}
|
||||
|
||||
pub fn load_history_entries(path: &Path, limit: usize, offset: usize) -> Result<Vec<HistoryEntry>, String> {
|
||||
let entries = read_all(path)?;
|
||||
Ok(entries.into_iter().skip(offset).take(limit).collect())
|
||||
}
|
||||
|
||||
pub fn clear_history_entries(path: &Path) -> Result<(), String> {
|
||||
write_all(path, &[])
|
||||
}
|
||||
|
||||
pub fn delete_history_entry_by_id(path: &Path, id: &str) -> Result<(), String> {
|
||||
let entries: Vec<HistoryEntry> = read_all(path)?
|
||||
.into_iter()
|
||||
.filter(|e| e.id != id)
|
||||
.collect();
|
||||
write_all(path, &entries)
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
pub mod ai;
|
||||
pub mod connection;
|
||||
pub mod connection_secrets;
|
||||
pub mod db;
|
||||
pub mod history;
|
||||
pub mod models;
|
||||
pub mod mongo_ops;
|
||||
pub mod query;
|
||||
pub mod query_cancel;
|
||||
pub mod redis_ops;
|
||||
pub mod schema;
|
||||
pub mod sql;
|
||||
pub mod table_import;
|
||||
pub mod transfer;
|
||||
pub mod types;
|
||||
pub mod update;
|
||||
@@ -0,0 +1,353 @@
|
||||
use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ConnectionConfig {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub db_type: DatabaseType,
|
||||
#[serde(default)]
|
||||
pub driver_profile: Option<String>,
|
||||
#[serde(default)]
|
||||
pub driver_label: Option<String>,
|
||||
#[serde(default)]
|
||||
pub url_params: Option<String>,
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
pub username: String,
|
||||
pub password: String,
|
||||
pub database: Option<String>,
|
||||
#[serde(default)]
|
||||
pub color: Option<String>,
|
||||
#[serde(default)]
|
||||
pub ssh_enabled: bool,
|
||||
#[serde(default)]
|
||||
pub ssh_host: String,
|
||||
#[serde(default = "default_ssh_port")]
|
||||
pub ssh_port: u16,
|
||||
#[serde(default)]
|
||||
pub ssh_user: String,
|
||||
#[serde(default)]
|
||||
pub ssh_password: String,
|
||||
#[serde(default)]
|
||||
pub ssh_key_path: String,
|
||||
#[serde(default)]
|
||||
pub ssh_key_passphrase: String,
|
||||
#[serde(default)]
|
||||
pub ssh_expose_lan: bool,
|
||||
#[serde(default)]
|
||||
pub ssl: bool,
|
||||
#[serde(default)]
|
||||
pub connection_string: Option<String>,
|
||||
}
|
||||
|
||||
fn default_ssh_port() -> u16 {
|
||||
22
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum DatabaseType {
|
||||
Mysql,
|
||||
Postgres,
|
||||
Sqlite,
|
||||
Redis,
|
||||
#[serde(rename = "duckdb")]
|
||||
DuckDb,
|
||||
#[serde(rename = "clickhouse")]
|
||||
ClickHouse,
|
||||
#[serde(rename = "sqlserver")]
|
||||
SqlServer,
|
||||
#[serde(rename = "mongodb")]
|
||||
MongoDb,
|
||||
#[serde(rename = "oracle")]
|
||||
Oracle,
|
||||
#[serde(rename = "elasticsearch")]
|
||||
Elasticsearch,
|
||||
Doris,
|
||||
#[serde(rename = "starrocks")]
|
||||
StarRocks,
|
||||
Redshift,
|
||||
}
|
||||
|
||||
impl ConnectionConfig {
|
||||
pub fn needs_bare_mysql(&self) -> bool {
|
||||
matches!(self.db_type, DatabaseType::Doris | DatabaseType::StarRocks)
|
||||
|| self.driver_profile.as_deref().map(|p| p.to_lowercase())
|
||||
.is_some_and(|p| matches!(p.as_str(), "doris" | "starrocks" | "selectdb" | "tdengine"))
|
||||
}
|
||||
|
||||
pub fn connection_url(&self) -> String {
|
||||
self.connection_url_with_host(&self.host, self.port)
|
||||
}
|
||||
|
||||
pub fn redacted_connection_url(&self) -> String {
|
||||
self.redacted_connection_url_with_host(&self.host, self.port)
|
||||
}
|
||||
|
||||
pub fn redacted_connection_url_with_host(&self, host: &str, port: u16) -> String {
|
||||
let host = bracket_ipv6(host);
|
||||
let db_part = self
|
||||
.database
|
||||
.as_deref()
|
||||
.filter(|d| !d.is_empty())
|
||||
.map(|d| format!("/{}", encode_url_part(d)))
|
||||
.unwrap_or_default();
|
||||
let params = self.normalized_url_params();
|
||||
|
||||
match self.db_type {
|
||||
DatabaseType::Sqlite | DatabaseType::DuckDb => {
|
||||
format!("{}?mode=rwc", self.host)
|
||||
}
|
||||
DatabaseType::Redis => {
|
||||
let scheme = if self.ssl { "rediss" } else { "redis" };
|
||||
format!("{scheme}://{host}:{port}/")
|
||||
}
|
||||
DatabaseType::Mysql | DatabaseType::Doris | DatabaseType::StarRocks => format!("mysql://{host}:{port}{db_part}?{params}"),
|
||||
DatabaseType::Postgres | DatabaseType::Redshift => {
|
||||
let suffix = if params.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("?{params}")
|
||||
};
|
||||
format!("postgres://{host}:{port}{db_part}{suffix}")
|
||||
}
|
||||
DatabaseType::ClickHouse => format!("http://{host}:{port}"),
|
||||
DatabaseType::SqlServer => format!(
|
||||
"server=tcp:{host},{port};database={}",
|
||||
self.database.as_deref().unwrap_or("master")
|
||||
),
|
||||
DatabaseType::MongoDb => {
|
||||
if let Some(cs) = self.connection_string.as_deref().filter(|s| !s.is_empty()) {
|
||||
return cs.to_string();
|
||||
}
|
||||
format!("mongodb://{host}:{port}{db_part}")
|
||||
}
|
||||
DatabaseType::Oracle => format!("oracle://{host}:{port}{db_part}"),
|
||||
DatabaseType::Elasticsearch => format!("http://{host}:{port}"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn connection_url_with_host(&self, host: &str, port: u16) -> String {
|
||||
let host = bracket_ipv6(host);
|
||||
let db_part = self
|
||||
.database
|
||||
.as_deref()
|
||||
.filter(|d| !d.is_empty())
|
||||
.map(|d| format!("/{}", encode_url_part(d)))
|
||||
.unwrap_or_default();
|
||||
let username = encode_url_part(&self.username);
|
||||
let password = encode_url_part(&self.password);
|
||||
let params = self.normalized_url_params();
|
||||
|
||||
match self.db_type {
|
||||
DatabaseType::Sqlite | DatabaseType::DuckDb => {
|
||||
format!("{}?mode=rwc", self.host)
|
||||
}
|
||||
DatabaseType::Redis => {
|
||||
let scheme = if self.ssl { "rediss" } else { "redis" };
|
||||
if self.username.is_empty() && self.password.is_empty() {
|
||||
format!("{scheme}://{host}:{port}/")
|
||||
} else if self.username.is_empty() {
|
||||
format!("{scheme}://:{password}@{host}:{port}/")
|
||||
} else {
|
||||
format!("{scheme}://{username}:{password}@{host}:{port}/")
|
||||
}
|
||||
}
|
||||
DatabaseType::Mysql | DatabaseType::Doris | DatabaseType::StarRocks => format!(
|
||||
"mysql://{}:{}@{host}:{port}{db_part}?{params}",
|
||||
username, password
|
||||
),
|
||||
DatabaseType::Postgres | DatabaseType::Redshift => {
|
||||
let suffix = if params.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("?{params}")
|
||||
};
|
||||
format!(
|
||||
"postgres://{}:{}@{host}:{port}{db_part}{suffix}",
|
||||
username, password
|
||||
)
|
||||
}
|
||||
DatabaseType::ClickHouse => format!("http://{host}:{port}"),
|
||||
DatabaseType::SqlServer => format!(
|
||||
"server=tcp:{host},{port};user={};password={};database={}",
|
||||
self.username,
|
||||
self.password,
|
||||
self.database.as_deref().unwrap_or("master")
|
||||
),
|
||||
DatabaseType::MongoDb => {
|
||||
if let Some(cs) = self.connection_string.as_deref().filter(|s| !s.is_empty()) {
|
||||
return cs.to_string();
|
||||
}
|
||||
if self.username.is_empty() {
|
||||
format!("mongodb://{host}:{port}{db_part}")
|
||||
} else {
|
||||
format!("mongodb://{username}:{password}@{host}:{port}{db_part}")
|
||||
}
|
||||
}
|
||||
DatabaseType::Oracle => {
|
||||
format!("oracle://{}:{}@{host}:{port}{db_part}", username, password)
|
||||
}
|
||||
DatabaseType::Elasticsearch => format!("http://{host}:{port}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn normalized_url_params(&self) -> String {
|
||||
let value = self.url_params.as_deref().unwrap_or("").trim();
|
||||
if self.needs_bare_mysql() {
|
||||
let v = value.trim_start_matches('?');
|
||||
let filtered: Vec<&str> = v.split('&')
|
||||
.filter(|p| !p.is_empty() && !p.starts_with("charset=") && !p.starts_with("ssl-mode=preferred"))
|
||||
.collect();
|
||||
return if filtered.is_empty() { "ssl-mode=disabled".to_string() } else { format!("ssl-mode=disabled&{}", filtered.join("&")) };
|
||||
}
|
||||
match self.db_type {
|
||||
DatabaseType::Mysql => {
|
||||
let base = "ssl-mode=preferred&charset=utf8mb4";
|
||||
if value.is_empty() {
|
||||
base.to_string()
|
||||
} else if value.contains("ssl-mode=") {
|
||||
let v = value.trim_start_matches('?');
|
||||
if v.contains("charset=") { v.to_string() } else { format!("{v}&charset=utf8mb4") }
|
||||
} else {
|
||||
let v = value.trim_start_matches('?');
|
||||
if v.contains("charset=") { format!("ssl-mode=preferred&{v}") } else { format!("{base}&{v}") }
|
||||
}
|
||||
}
|
||||
DatabaseType::Doris | DatabaseType::StarRocks => {
|
||||
let v = value.trim_start_matches('?');
|
||||
let filtered: Vec<&str> = v.split('&')
|
||||
.filter(|p| !p.is_empty() && !p.starts_with("charset=") && !p.starts_with("ssl-mode=preferred"))
|
||||
.collect();
|
||||
if filtered.is_empty() { "ssl-mode=disabled".to_string() } else { format!("ssl-mode=disabled&{}", filtered.join("&")) }
|
||||
}
|
||||
DatabaseType::Postgres | DatabaseType::Redshift => value.trim_start_matches('?').to_string(),
|
||||
_ => value.trim_start_matches('?').to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_url_part(value: &str) -> String {
|
||||
utf8_percent_encode(value, NON_ALPHANUMERIC).to_string()
|
||||
}
|
||||
|
||||
fn bracket_ipv6(host: &str) -> String {
|
||||
if host.contains(':') && !host.starts_with('[') {
|
||||
format!("[{host}]")
|
||||
} else {
|
||||
host.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{ConnectionConfig, DatabaseType};
|
||||
|
||||
fn mysql_config(username: &str, password: &str, database: Option<&str>) -> ConnectionConfig {
|
||||
ConnectionConfig {
|
||||
id: "id".to_string(),
|
||||
name: "name".to_string(),
|
||||
db_type: DatabaseType::Mysql,
|
||||
driver_profile: None,
|
||||
driver_label: None,
|
||||
url_params: None,
|
||||
host: "10.1.2.3".to_string(),
|
||||
port: 2883,
|
||||
username: username.to_string(),
|
||||
password: password.to_string(),
|
||||
database: database.map(str::to_string),
|
||||
color: None,
|
||||
ssh_enabled: false,
|
||||
ssh_host: String::new(),
|
||||
ssh_port: 22,
|
||||
ssh_user: String::new(),
|
||||
ssh_password: String::new(),
|
||||
ssh_key_path: String::new(),
|
||||
ssh_key_passphrase: String::new(),
|
||||
ssh_expose_lan: false,
|
||||
ssl: false,
|
||||
connection_string: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_url_encodes_oceanbase_username() {
|
||||
let config = mysql_config("user@tenant#cluster", "secret", None);
|
||||
|
||||
assert_eq!(
|
||||
config.connection_url(),
|
||||
"mysql://user%40tenant%23cluster:secret@10.1.2.3:2883?ssl-mode=preferred&charset=utf8mb4"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_url_encodes_password_and_database() {
|
||||
let config = mysql_config("root", "p@ss:word#1", Some("db/name"));
|
||||
|
||||
assert_eq!(
|
||||
config.connection_url(),
|
||||
"mysql://root:p%40ss%3Aword%231@10.1.2.3:2883/db%2Fname?ssl-mode=preferred&charset=utf8mb4"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_url_appends_custom_params() {
|
||||
let mut config = mysql_config("root", "secret", Some("test"));
|
||||
config.url_params = Some("charset=utf8mb4".to_string());
|
||||
|
||||
assert_eq!(
|
||||
config.connection_url(),
|
||||
"mysql://root:secret@10.1.2.3:2883/test?ssl-mode=preferred&charset=utf8mb4"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn postgres_url_appends_custom_params() {
|
||||
let mut config = mysql_config("postgres", "secret", Some("test"));
|
||||
config.db_type = DatabaseType::Postgres;
|
||||
config.url_params = Some("sslmode=disable".to_string());
|
||||
|
||||
assert_eq!(
|
||||
config.connection_url(),
|
||||
"postgres://postgres:secret@10.1.2.3:2883/test?sslmode=disable"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redacted_mysql_url_omits_credentials() {
|
||||
let config = mysql_config("user@tenant#cluster", "p@ss:word#1", Some("db/name"));
|
||||
|
||||
let url = config.redacted_connection_url();
|
||||
|
||||
assert_eq!(url, "mysql://10.1.2.3:2883/db%2Fname?ssl-mode=preferred&charset=utf8mb4");
|
||||
assert!(!url.contains("user"));
|
||||
assert!(!url.contains("p%40ss"));
|
||||
assert!(!url.contains("p@ss"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redacted_sqlserver_url_omits_credentials() {
|
||||
let mut config = mysql_config("sa", "super-secret", Some("master"));
|
||||
config.db_type = DatabaseType::SqlServer;
|
||||
|
||||
let url = config.redacted_connection_url();
|
||||
|
||||
assert_eq!(url, "server=tcp:10.1.2.3,2883;database=master");
|
||||
assert!(!url.contains("sa"));
|
||||
assert!(!url.contains("super-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redacted_redis_url_omits_credentials_and_keeps_tls_scheme() {
|
||||
let mut config = mysql_config("default", "redis-secret", None);
|
||||
config.db_type = DatabaseType::Redis;
|
||||
config.ssl = true;
|
||||
|
||||
let url = config.redacted_connection_url();
|
||||
|
||||
assert_eq!(url, "rediss://10.1.2.3:2883/");
|
||||
assert!(!url.contains("default"));
|
||||
assert!(!url.contains("redis-secret"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
pub mod connection;
|
||||
@@ -0,0 +1,114 @@
|
||||
use crate::connection::{AppState, PoolKind};
|
||||
use crate::db::mongo_driver::{self, MongoDocumentResult};
|
||||
use crate::db::elasticsearch_driver;
|
||||
|
||||
pub async fn mongo_list_databases_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
) -> Result<Vec<String>, String> {
|
||||
let connections = state.connections.lock().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => mongo_driver::list_databases(client).await,
|
||||
PoolKind::Elasticsearch(_) => Ok(vec!["default".to_string()]),
|
||||
_ => Err("Not a MongoDB/Elasticsearch connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn mongo_list_collections_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
) -> Result<Vec<String>, String> {
|
||||
let connections = state.connections.lock().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => mongo_driver::list_collections(client, database).await,
|
||||
PoolKind::Elasticsearch(client) => elasticsearch_driver::list_indices(client).await,
|
||||
_ => Err("Not a MongoDB/Elasticsearch connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn mongo_find_documents_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
collection: &str,
|
||||
skip: u64,
|
||||
limit: i64,
|
||||
) -> Result<MongoDocumentResult, String> {
|
||||
let connections = state.connections.lock().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => {
|
||||
mongo_driver::find_documents(client, database, collection, skip, limit).await
|
||||
}
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
elasticsearch_driver::find_documents(&client, collection, skip, limit).await
|
||||
}
|
||||
_ => Err("Not a MongoDB/Elasticsearch connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn mongo_insert_document_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
collection: &str,
|
||||
doc_json: &str,
|
||||
) -> Result<String, String> {
|
||||
let connections = state.connections.lock().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => {
|
||||
mongo_driver::insert_document(client, database, collection, doc_json).await
|
||||
}
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
elasticsearch_driver::insert_document(&client, collection, doc_json).await
|
||||
}
|
||||
_ => Err("Not a MongoDB/Elasticsearch connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn mongo_update_document_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
collection: &str,
|
||||
id: &str,
|
||||
doc_json: &str,
|
||||
) -> Result<u64, String> {
|
||||
let connections = state.connections.lock().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => {
|
||||
mongo_driver::update_document(client, database, collection, id, doc_json).await
|
||||
}
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
elasticsearch_driver::update_document(&client, collection, id, doc_json).await
|
||||
}
|
||||
_ => Err("Not a MongoDB/Elasticsearch connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn mongo_delete_document_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
collection: &str,
|
||||
id: &str,
|
||||
) -> Result<u64, String> {
|
||||
let connections = state.connections.lock().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => {
|
||||
mongo_driver::delete_document(client, database, collection, id).await
|
||||
}
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
elasticsearch_driver::delete_document(&client, collection, id).await
|
||||
}
|
||||
_ => Err("Not a MongoDB/Elasticsearch connection".to_string()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,366 @@
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::connection::{AppState, PoolKind};
|
||||
use crate::db;
|
||||
use crate::sql::split_sql_statements;
|
||||
|
||||
pub const QUERY_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
pub const MAX_ROWS: usize = 10000;
|
||||
pub const QUERY_CANCELED: &str = "Query canceled";
|
||||
|
||||
pub fn duckdb_execute(con: &duckdb::Connection, sql: &str) -> Result<db::QueryResult, String> {
|
||||
let start = std::time::Instant::now();
|
||||
let trimmed = sql.trim().to_uppercase();
|
||||
|
||||
if trimmed.starts_with("SELECT") || trimmed.starts_with("SHOW") || trimmed.starts_with("DESCRIBE")
|
||||
|| trimmed.starts_with("EXPLAIN") || trimmed.starts_with("WITH") || trimmed.starts_with("PRAGMA")
|
||||
{
|
||||
let mut stmt = con.prepare(sql).map_err(|e| e.to_string())?;
|
||||
let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
|
||||
let stmt_ref = rows.as_ref().ok_or("DuckDB statement unavailable")?;
|
||||
let col_count = stmt_ref.column_count();
|
||||
let columns: Vec<String> = (0..col_count)
|
||||
.map(|i| stmt_ref.column_name(i).map(|s| s.to_string()).unwrap_or_else(|_| "?".to_string()))
|
||||
.collect();
|
||||
|
||||
let mut result_rows = Vec::new();
|
||||
while let Some(row) = rows.next().map_err(|e| e.to_string())? {
|
||||
if result_rows.len() >= MAX_ROWS { break; }
|
||||
let vals: Vec<serde_json::Value> = (0..col_count).map(|i| {
|
||||
row.get::<_, String>(i)
|
||||
.map(serde_json::Value::String)
|
||||
.or_else(|_| row.get::<_, i64>(i).map(|v| serde_json::Value::Number(v.into())))
|
||||
.or_else(|_| row.get::<_, f64>(i).map(|v| {
|
||||
serde_json::Number::from_f64(v)
|
||||
.map(serde_json::Value::Number)
|
||||
.unwrap_or(serde_json::Value::Null)
|
||||
}))
|
||||
.or_else(|_| row.get::<_, bool>(i).map(serde_json::Value::Bool))
|
||||
.unwrap_or(serde_json::Value::Null)
|
||||
}).collect();
|
||||
result_rows.push(vals);
|
||||
}
|
||||
|
||||
let truncated = result_rows.len() >= MAX_ROWS;
|
||||
Ok(db::QueryResult { columns, rows: result_rows, affected_rows: 0, execution_time_ms: start.elapsed().as_millis(), truncated })
|
||||
} else {
|
||||
let affected = con.execute(sql, []).map_err(|e| e.to_string())?;
|
||||
Ok(db::QueryResult { columns: vec![], rows: vec![], affected_rows: affected as u64, execution_time_ms: start.elapsed().as_millis(), truncated: false })
|
||||
}
|
||||
}
|
||||
|
||||
pub fn truncate_result(mut result: db::QueryResult) -> db::QueryResult {
|
||||
if result.rows.len() > MAX_ROWS {
|
||||
result.rows.truncate(MAX_ROWS);
|
||||
result.truncated = true;
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub fn is_connection_error(err: &str) -> bool {
|
||||
let lower = err.to_lowercase();
|
||||
lower.contains("connection")
|
||||
|| lower.contains("broken pipe")
|
||||
|| lower.contains("reset by peer")
|
||||
|| lower.contains("timed out")
|
||||
|| lower.contains("closed")
|
||||
|| lower.contains("eof")
|
||||
}
|
||||
|
||||
pub fn timeout_error() -> String {
|
||||
format!("Query timed out after {} seconds", QUERY_TIMEOUT.as_secs())
|
||||
}
|
||||
|
||||
pub fn canceled_error() -> String {
|
||||
QUERY_CANCELED.to_string()
|
||||
}
|
||||
|
||||
pub fn is_canceled(cancel_token: &Option<CancellationToken>) -> bool {
|
||||
cancel_token
|
||||
.as_ref()
|
||||
.map(|token| token.is_cancelled())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
pub async fn wait_for_query<F>(
|
||||
cancel_token: Option<CancellationToken>,
|
||||
future: F,
|
||||
) -> Result<db::QueryResult, String>
|
||||
where
|
||||
F: Future<Output = Result<db::QueryResult, String>>,
|
||||
{
|
||||
wait_for_query_with_timeout(cancel_token, QUERY_TIMEOUT, future).await
|
||||
}
|
||||
|
||||
pub async fn wait_for_query_with_timeout<F>(
|
||||
cancel_token: Option<CancellationToken>,
|
||||
timeout_duration: Duration,
|
||||
future: F,
|
||||
) -> Result<db::QueryResult, String>
|
||||
where
|
||||
F: Future<Output = Result<db::QueryResult, String>>,
|
||||
{
|
||||
if let Some(token) = cancel_token {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => Err(canceled_error()),
|
||||
result = timeout(timeout_duration, future) => result.map_err(|_| timeout_error())?,
|
||||
}
|
||||
} else {
|
||||
timeout(timeout_duration, future)
|
||||
.await
|
||||
.map_err(|_| timeout_error())?
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn do_execute(
|
||||
state: &AppState,
|
||||
pool_key: &str,
|
||||
sql: &str,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
) -> Result<db::QueryResult, String> {
|
||||
let connections = state.connections.lock().await;
|
||||
let pool = connections.get(pool_key).ok_or("Connection not found")?;
|
||||
|
||||
match pool {
|
||||
PoolKind::DuckDb(con) => {
|
||||
let con = con.clone();
|
||||
let sql = sql.to_string();
|
||||
drop(connections);
|
||||
wait_for_query(cancel_token, async move {
|
||||
let task = tokio::task::spawn_blocking(move || {
|
||||
let con = con.lock().map_err(|e| e.to_string())?;
|
||||
duckdb_execute(&con, &sql)
|
||||
});
|
||||
task.await.map_err(|e| e.to_string())?
|
||||
})
|
||||
.await
|
||||
}
|
||||
PoolKind::Mysql(p, bare) => {
|
||||
let p = p.clone();
|
||||
let bare = *bare;
|
||||
drop(connections);
|
||||
wait_for_query(cancel_token, db::mysql::execute_query(&p, sql, bare))
|
||||
.await
|
||||
.map(truncate_result)
|
||||
}
|
||||
PoolKind::Postgres(p) => {
|
||||
let p = p.clone();
|
||||
drop(connections);
|
||||
wait_for_query(cancel_token, db::postgres::execute_query(&p, sql))
|
||||
.await
|
||||
.map(truncate_result)
|
||||
}
|
||||
PoolKind::Sqlite(p) => {
|
||||
let p = p.clone();
|
||||
drop(connections);
|
||||
wait_for_query(cancel_token, db::sqlite::execute_query(&p, sql))
|
||||
.await
|
||||
.map(truncate_result)
|
||||
}
|
||||
PoolKind::ClickHouse(client) => {
|
||||
let client = client.clone();
|
||||
let database = pool_key.split(':').nth(1).unwrap_or("default").to_string();
|
||||
drop(connections);
|
||||
wait_for_query(cancel_token, db::clickhouse_driver::execute_query(&client, &database, sql))
|
||||
.await
|
||||
.map(truncate_result)
|
||||
}
|
||||
PoolKind::SqlServer(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
let mut client = match cancel_token.as_ref() {
|
||||
Some(token) => tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => return Err(canceled_error()),
|
||||
guard = client.lock() => guard,
|
||||
},
|
||||
None => client.lock().await,
|
||||
};
|
||||
wait_for_query(cancel_token, db::sqlserver::execute_query(&mut client, sql))
|
||||
.await
|
||||
.map(truncate_result)
|
||||
}
|
||||
PoolKind::Oracle(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
let client = match cancel_token.as_ref() {
|
||||
Some(token) => tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => return Err(canceled_error()),
|
||||
guard = client.lock() => guard,
|
||||
},
|
||||
None => client.lock().await,
|
||||
};
|
||||
wait_for_query(cancel_token, db::oracle_driver::execute_query(&*client, sql))
|
||||
.await
|
||||
.map(truncate_result)
|
||||
}
|
||||
PoolKind::Elasticsearch(_) => Err("Use document browser for Elasticsearch".to_string()),
|
||||
PoolKind::Redis(_) => Err("Use Redis-specific commands".to_string()),
|
||||
PoolKind::MongoDb(_) => Err("Use MongoDB-specific commands".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn execute_sql_statement(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
sql: &str,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
) -> Result<db::QueryResult, String> {
|
||||
let pool_key = if database.is_empty() {
|
||||
connection_id.to_string()
|
||||
} else {
|
||||
state.get_or_create_pool(connection_id, Some(database)).await?
|
||||
};
|
||||
|
||||
if is_canceled(&cancel_token) {
|
||||
return Err(canceled_error());
|
||||
}
|
||||
|
||||
let result = do_execute(state, &pool_key, sql, cancel_token.clone()).await;
|
||||
|
||||
match &result {
|
||||
Err(e) if is_connection_error(e) && !is_canceled(&cancel_token) => {
|
||||
let db_opt = if database.is_empty() { None } else { Some(database) };
|
||||
let new_key = state.reconnect_pool(connection_id, db_opt).await?;
|
||||
do_execute(state, &new_key, sql, cancel_token).await
|
||||
}
|
||||
_ => result,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn execute_multi_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
sql: &str,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
) -> Result<Vec<db::QueryResult>, String> {
|
||||
let statements = split_sql_statements(sql);
|
||||
if statements.len() <= 1 {
|
||||
let single_sql = statements.into_iter().next().unwrap_or_default();
|
||||
let result = execute_sql_statement(
|
||||
state, connection_id, database, &single_sql, cancel_token,
|
||||
).await?;
|
||||
return Ok(vec![result]);
|
||||
}
|
||||
|
||||
let mut results = Vec::with_capacity(statements.len());
|
||||
for stmt in &statements {
|
||||
if is_canceled(&cancel_token) {
|
||||
results.push(db::QueryResult {
|
||||
columns: vec!["Error".to_string()],
|
||||
rows: vec![vec![serde_json::Value::String(canceled_error())]],
|
||||
affected_rows: 0,
|
||||
execution_time_ms: 0,
|
||||
truncated: false,
|
||||
});
|
||||
break;
|
||||
}
|
||||
match execute_sql_statement(
|
||||
state, connection_id, database, stmt, cancel_token.clone(),
|
||||
).await {
|
||||
Ok(r) => results.push(r),
|
||||
Err(e) => {
|
||||
results.push(db::QueryResult {
|
||||
columns: vec!["Error".to_string()],
|
||||
rows: vec![vec![serde_json::Value::String(e)]],
|
||||
affected_rows: 0,
|
||||
execution_time_ms: 0,
|
||||
truncated: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
pub async fn execute_statements(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
statements: &[String],
|
||||
) -> Result<db::QueryResult, String> {
|
||||
let pool_key = if database.is_empty() {
|
||||
connection_id.to_string()
|
||||
} else {
|
||||
state.get_or_create_pool(connection_id, Some(database)).await?
|
||||
};
|
||||
|
||||
let mut total_affected: u64 = 0;
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
for (i, sql) in statements.iter().enumerate() {
|
||||
match do_execute(state, &pool_key, sql, None).await {
|
||||
Ok(result) => {
|
||||
total_affected += result.affected_rows;
|
||||
}
|
||||
Err(e) => {
|
||||
if is_connection_error(&e) {
|
||||
let db_opt = if database.is_empty() { None } else { Some(database) };
|
||||
let _ = state.reconnect_pool(connection_id, db_opt).await;
|
||||
}
|
||||
return Err(format!(
|
||||
"Statement {} failed: {}. Previous {} statement(s) may have been committed.",
|
||||
i + 1, e, i
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(db::QueryResult {
|
||||
columns: vec![],
|
||||
rows: vec![],
|
||||
affected_rows: total_affected,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn wait_for_query_returns_cancelled_when_token_is_cancelled() {
|
||||
let token = CancellationToken::new();
|
||||
token.cancel();
|
||||
|
||||
let result = wait_for_query(Some(token), async {
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
Ok(db::QueryResult {
|
||||
columns: vec![],
|
||||
rows: vec![],
|
||||
affected_rows: 0,
|
||||
execution_time_ms: 0,
|
||||
truncated: false,
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(result.unwrap_err(), QUERY_CANCELED);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn wait_for_query_without_token_still_times_out() {
|
||||
let result = wait_for_query_with_timeout(None, Duration::from_millis(10), async {
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
Ok(db::QueryResult {
|
||||
columns: vec![],
|
||||
rows: vec![],
|
||||
affected_rows: 0,
|
||||
execution_time_ms: 0,
|
||||
truncated: false,
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(result.unwrap_err(), timeout_error());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct RunningQueries {
|
||||
inner: Arc<Mutex<HashMap<String, CancellationToken>>>,
|
||||
}
|
||||
|
||||
impl RunningQueries {
|
||||
pub fn register(&self, execution_id: String) -> RegisteredQuery {
|
||||
let token = CancellationToken::new();
|
||||
self.inner
|
||||
.lock()
|
||||
.expect("running query registry poisoned")
|
||||
.insert(execution_id.clone(), token.clone());
|
||||
|
||||
RegisteredQuery {
|
||||
execution_id,
|
||||
token,
|
||||
running_queries: self.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn cancel(&self, execution_id: &str) -> bool {
|
||||
let token = self
|
||||
.inner
|
||||
.lock()
|
||||
.expect("running query registry poisoned")
|
||||
.get(execution_id)
|
||||
.cloned();
|
||||
|
||||
if let Some(token) = token {
|
||||
token.cancel();
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn has(&self, execution_id: &str) -> bool {
|
||||
self.inner
|
||||
.lock()
|
||||
.expect("running query registry poisoned")
|
||||
.contains_key(execution_id)
|
||||
}
|
||||
|
||||
fn remove(&self, execution_id: &str) {
|
||||
self.inner
|
||||
.lock()
|
||||
.expect("running query registry poisoned")
|
||||
.remove(execution_id);
|
||||
}
|
||||
}
|
||||
|
||||
pub struct RegisteredQuery {
|
||||
execution_id: String,
|
||||
token: CancellationToken,
|
||||
running_queries: RunningQueries,
|
||||
}
|
||||
|
||||
impl RegisteredQuery {
|
||||
pub fn token(&self) -> CancellationToken {
|
||||
self.token.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RegisteredQuery {
|
||||
fn drop(&mut self) {
|
||||
self.running_queries.remove(&self.execution_id);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::RunningQueries;
|
||||
|
||||
#[test]
|
||||
fn cancel_marks_registered_query_as_cancelled() {
|
||||
let running = RunningQueries::default();
|
||||
let registered = running.register("exec-1".to_string());
|
||||
|
||||
assert!(running.cancel("exec-1"));
|
||||
assert!(registered.token().is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dropping_registration_removes_running_query() {
|
||||
let running = RunningQueries::default();
|
||||
let registered = running.register("exec-1".to_string());
|
||||
|
||||
assert!(running.has("exec-1"));
|
||||
drop(registered);
|
||||
|
||||
assert!(!running.has("exec-1"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
use crate::connection::{AppState, PoolKind};
|
||||
use crate::db::redis_driver::{self, RedisScanResult, RedisValue};
|
||||
|
||||
pub async fn redis_list_databases_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
) -> Result<Vec<u32>, String> {
|
||||
let connections = state.connections.lock().await;
|
||||
let pool = connections.get(connection_id).ok_or("Connection not found")?;
|
||||
match pool {
|
||||
PoolKind::Redis(con) => {
|
||||
let mut con = con.lock().await;
|
||||
redis_driver::list_databases(&mut con).await
|
||||
}
|
||||
_ => Err("Not a Redis connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn redis_scan_keys_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
db: u32,
|
||||
cursor: u64,
|
||||
pattern: &str,
|
||||
count: usize,
|
||||
) -> Result<RedisScanResult, String> {
|
||||
let connections = state.connections.lock().await;
|
||||
let pool = connections.get(connection_id).ok_or("Connection not found")?;
|
||||
match pool {
|
||||
PoolKind::Redis(con) => {
|
||||
let mut con = con.lock().await;
|
||||
redis_driver::select_db(&mut con, db).await?;
|
||||
redis_driver::scan_keys_page(&mut con, cursor, pattern, count).await
|
||||
}
|
||||
_ => Err("Not a Redis connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn redis_get_value_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
key: &str,
|
||||
) -> Result<RedisValue, String> {
|
||||
let connections = state.connections.lock().await;
|
||||
let pool = connections.get(connection_id).ok_or("Connection not found")?;
|
||||
match pool {
|
||||
PoolKind::Redis(con) => {
|
||||
let mut con = con.lock().await;
|
||||
redis_driver::get_value(&mut con, key).await
|
||||
}
|
||||
_ => Err("Not a Redis connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn redis_set_string_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
key: &str,
|
||||
value: &str,
|
||||
ttl: Option<i64>,
|
||||
) -> Result<(), String> {
|
||||
let connections = state.connections.lock().await;
|
||||
let pool = connections.get(connection_id).ok_or("Connection not found")?;
|
||||
match pool {
|
||||
PoolKind::Redis(con) => {
|
||||
let mut con = con.lock().await;
|
||||
redis_driver::set_string(&mut con, key, value, ttl).await
|
||||
}
|
||||
_ => Err("Not a Redis connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn redis_delete_key_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
key: &str,
|
||||
) -> Result<(), String> {
|
||||
let connections = state.connections.lock().await;
|
||||
let pool = connections.get(connection_id).ok_or("Connection not found")?;
|
||||
match pool {
|
||||
PoolKind::Redis(con) => {
|
||||
let mut con = con.lock().await;
|
||||
redis_driver::delete_key(&mut con, key).await
|
||||
}
|
||||
_ => Err("Not a Redis connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn redis_hash_set_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
key: &str,
|
||||
field: &str,
|
||||
value: &str,
|
||||
) -> Result<(), String> {
|
||||
let connections = state.connections.lock().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::Redis(con) => redis_driver::hash_set(&mut *con.lock().await, key, field, value).await,
|
||||
_ => Err("Not a Redis connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn redis_hash_del_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
key: &str,
|
||||
field: &str,
|
||||
) -> Result<(), String> {
|
||||
let connections = state.connections.lock().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::Redis(con) => redis_driver::hash_del(&mut *con.lock().await, key, field).await,
|
||||
_ => Err("Not a Redis connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn redis_list_push_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
key: &str,
|
||||
value: &str,
|
||||
) -> Result<(), String> {
|
||||
let connections = state.connections.lock().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::Redis(con) => redis_driver::list_push(&mut *con.lock().await, key, value).await,
|
||||
_ => Err("Not a Redis connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn redis_list_remove_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
key: &str,
|
||||
index: i64,
|
||||
) -> Result<(), String> {
|
||||
let connections = state.connections.lock().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::Redis(con) => redis_driver::list_remove(&mut *con.lock().await, key, index).await,
|
||||
_ => Err("Not a Redis connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn redis_set_add_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
key: &str,
|
||||
member: &str,
|
||||
) -> Result<(), String> {
|
||||
let connections = state.connections.lock().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::Redis(con) => redis_driver::set_add(&mut *con.lock().await, key, member).await,
|
||||
_ => Err("Not a Redis connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn redis_set_remove_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
key: &str,
|
||||
member: &str,
|
||||
) -> Result<(), String> {
|
||||
let connections = state.connections.lock().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::Redis(con) => redis_driver::set_remove(&mut *con.lock().await, key, member).await,
|
||||
_ => Err("Not a Redis connection".to_string()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,515 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::connection::{AppState, PoolKind};
|
||||
use crate::db;
|
||||
|
||||
pub fn duckdb_query_tables(con: &duckdb::Connection) -> Result<Vec<db::TableInfo>, String> {
|
||||
let mut stmt = con.prepare(
|
||||
"SELECT table_name, table_type FROM information_schema.tables WHERE table_schema = 'main' ORDER BY table_name"
|
||||
).map_err(|e| e.to_string())?;
|
||||
let rows = stmt.query_map([], |row| {
|
||||
Ok(db::TableInfo {
|
||||
name: row.get::<_, String>(0)?,
|
||||
table_type: row.get::<_, String>(1)?,
|
||||
})
|
||||
}).map_err(|e| e.to_string())?;
|
||||
Ok(rows.filter_map(|r| r.ok()).collect())
|
||||
}
|
||||
|
||||
pub fn duckdb_query_columns(con: &duckdb::Connection, table: &str) -> Result<Vec<db::ColumnInfo>, String> {
|
||||
let mut pk_stmt = con.prepare(
|
||||
"SELECT kcu.column_name
|
||||
FROM information_schema.table_constraints tc
|
||||
JOIN information_schema.key_column_usage kcu
|
||||
ON tc.constraint_name = kcu.constraint_name
|
||||
AND tc.table_schema = kcu.table_schema
|
||||
AND tc.table_name = kcu.table_name
|
||||
WHERE tc.constraint_type = 'PRIMARY KEY'
|
||||
AND tc.table_schema = 'main'
|
||||
AND tc.table_name = ?
|
||||
ORDER BY kcu.ordinal_position"
|
||||
).map_err(|e| e.to_string())?;
|
||||
let pk_rows = pk_stmt.query_map([table], |row| row.get::<_, String>(0))
|
||||
.map_err(|e| e.to_string())?;
|
||||
let primary_keys: std::collections::HashSet<String> = pk_rows.filter_map(|r| r.ok()).collect();
|
||||
|
||||
let mut stmt = con.prepare(
|
||||
"SELECT column_name, data_type, is_nullable, column_default
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'main' AND table_name = ?
|
||||
ORDER BY ordinal_position"
|
||||
).map_err(|e| e.to_string())?;
|
||||
let rows = stmt.query_map([table], |row| {
|
||||
let name = row.get::<_, String>(0)?;
|
||||
Ok(db::ColumnInfo {
|
||||
is_primary_key: primary_keys.contains(&name),
|
||||
name,
|
||||
data_type: row.get::<_, String>(1)?,
|
||||
is_nullable: row.get::<_, String>(2).unwrap_or_default() == "YES",
|
||||
column_default: row.get::<_, Option<String>>(3)?,
|
||||
extra: None, comment: None,
|
||||
numeric_precision: None,
|
||||
numeric_scale: None,
|
||||
character_maximum_length: None,
|
||||
})
|
||||
}).map_err(|e| e.to_string())?;
|
||||
Ok(rows.filter_map(|r| r.ok()).collect())
|
||||
}
|
||||
|
||||
pub fn extract_duckdb(connections: &HashMap<String, PoolKind>, key: &str) -> Option<Arc<std::sync::Mutex<duckdb::Connection>>> {
|
||||
match connections.get(key)? {
|
||||
PoolKind::DuckDb(con) => Some(con.clone()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn extract_sqlserver(connections: &HashMap<String, PoolKind>, key: &str) -> Option<Arc<tokio::sync::Mutex<db::sqlserver::SqlServerClient>>> {
|
||||
match connections.get(key)? {
|
||||
PoolKind::SqlServer(client) => Some(client.clone()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn extract_clickhouse(connections: &HashMap<String, PoolKind>, key: &str) -> Option<db::clickhouse_driver::ChClient> {
|
||||
match connections.get(key)? {
|
||||
PoolKind::ClickHouse(client) => Some(client.clone()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn extract_oracle(connections: &HashMap<String, PoolKind>, key: &str) -> Option<Arc<tokio::sync::Mutex<db::oracle_driver::OracleClient>>> {
|
||||
match connections.get(key)? {
|
||||
PoolKind::Oracle(client) => Some(client.clone()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn list_databases_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
) -> Result<Vec<db::DatabaseInfo>, String> {
|
||||
{
|
||||
let connections = state.connections.lock().await;
|
||||
if let Some(client) = extract_clickhouse(&connections, connection_id) {
|
||||
drop(connections);
|
||||
return db::clickhouse_driver::list_databases(&client).await;
|
||||
}
|
||||
if let Some(client) = extract_sqlserver(&connections, connection_id) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
return db::sqlserver::list_databases(&mut client).await;
|
||||
}
|
||||
if let Some(client) = extract_oracle(&connections, connection_id) {
|
||||
drop(connections);
|
||||
let client = client.lock().await;
|
||||
return db::oracle_driver::list_databases(&*client).await;
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.lock().await;
|
||||
let pool = connections.get(connection_id).ok_or("Connection not found")?;
|
||||
|
||||
match pool {
|
||||
PoolKind::Mysql(p, _) => db::mysql::list_databases(p).await,
|
||||
PoolKind::Postgres(p) => db::postgres::list_databases(p).await,
|
||||
PoolKind::Sqlite(p) => db::sqlite::list_databases(p).await,
|
||||
PoolKind::DuckDb(_) => Ok(vec![db::DatabaseInfo { name: "main".to_string() }]),
|
||||
_ => Ok(vec![]),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn list_schemas_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
) -> Result<Vec<String>, String> {
|
||||
let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?;
|
||||
|
||||
{
|
||||
let connections = state.connections.lock().await;
|
||||
if let Some(client) = extract_sqlserver(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
return db::sqlserver::list_schemas(&mut client).await;
|
||||
}
|
||||
if let Some(client) = extract_oracle(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
let client = client.lock().await;
|
||||
return db::oracle_driver::list_schemas(&*client).await;
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.lock().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
PoolKind::Postgres(p) => db::postgres::list_schemas(p).await,
|
||||
_ => Ok(vec![]),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn list_tables_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
schema: &str,
|
||||
) -> Result<Vec<db::TableInfo>, String> {
|
||||
let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?;
|
||||
|
||||
{
|
||||
let connections = state.connections.lock().await;
|
||||
if let Some(con) = extract_duckdb(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
let con = con.lock().map_err(|e| e.to_string())?;
|
||||
return duckdb_query_tables(&con);
|
||||
}
|
||||
if let Some(client) = extract_clickhouse(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
return db::clickhouse_driver::list_tables(&client, database).await;
|
||||
}
|
||||
if let Some(client) = extract_sqlserver(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
return db::sqlserver::list_tables(&mut client, schema).await;
|
||||
}
|
||||
if let Some(client) = extract_oracle(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
let client = client.lock().await;
|
||||
return db::oracle_driver::list_tables(&*client, schema).await;
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.lock().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
PoolKind::Mysql(p, _) => db::mysql::list_tables(p, schema).await,
|
||||
PoolKind::Postgres(p) => db::postgres::list_tables(p, schema).await,
|
||||
PoolKind::Sqlite(p) => db::sqlite::list_tables(p, schema).await,
|
||||
_ => Ok(vec![]),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_columns_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
schema: &str,
|
||||
table: &str,
|
||||
) -> Result<Vec<db::ColumnInfo>, String> {
|
||||
let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?;
|
||||
|
||||
{
|
||||
let connections = state.connections.lock().await;
|
||||
if let Some(con) = extract_duckdb(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
let con = con.lock().map_err(|e| e.to_string())?;
|
||||
return duckdb_query_columns(&con, table);
|
||||
}
|
||||
if let Some(client) = extract_clickhouse(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
return db::clickhouse_driver::get_columns(&client, database, table).await;
|
||||
}
|
||||
if let Some(client) = extract_sqlserver(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
return db::sqlserver::get_columns(&mut client, schema, table).await;
|
||||
}
|
||||
if let Some(client) = extract_oracle(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
let client = client.lock().await;
|
||||
return db::oracle_driver::get_columns(&*client, schema, table).await;
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.lock().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
PoolKind::Mysql(p, _) => db::mysql::get_columns(p, schema, table).await,
|
||||
PoolKind::Postgres(p) => db::postgres::get_columns(p, schema, table).await,
|
||||
PoolKind::Sqlite(p) => db::sqlite::get_columns(p, schema, table).await,
|
||||
_ => Ok(vec![]),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn list_indexes_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
schema: &str,
|
||||
table: &str,
|
||||
) -> Result<Vec<db::IndexInfo>, String> {
|
||||
let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?;
|
||||
|
||||
{
|
||||
let connections = state.connections.lock().await;
|
||||
if let Some(client) = extract_sqlserver(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
return db::sqlserver::list_indexes(&mut client, schema, table).await;
|
||||
}
|
||||
if let Some(client) = extract_oracle(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
let client = client.lock().await;
|
||||
return db::oracle_driver::list_indexes(&*client, schema, table).await;
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.lock().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
PoolKind::Mysql(p, _) => db::mysql::list_indexes(p, schema, table).await,
|
||||
PoolKind::Postgres(p) => db::postgres::list_indexes(p, schema, table).await,
|
||||
PoolKind::Sqlite(p) => db::sqlite::list_indexes(p, schema, table).await,
|
||||
_ => Ok(vec![]),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn list_foreign_keys_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
schema: &str,
|
||||
table: &str,
|
||||
) -> Result<Vec<db::ForeignKeyInfo>, String> {
|
||||
let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?;
|
||||
|
||||
{
|
||||
let connections = state.connections.lock().await;
|
||||
if let Some(client) = extract_sqlserver(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
return db::sqlserver::list_foreign_keys(&mut client, schema, table).await;
|
||||
}
|
||||
if let Some(client) = extract_oracle(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
let client = client.lock().await;
|
||||
return db::oracle_driver::list_foreign_keys(&*client, schema, table).await;
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.lock().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
PoolKind::Mysql(p, _) => db::mysql::list_foreign_keys(p, schema, table).await,
|
||||
PoolKind::Postgres(p) => db::postgres::list_foreign_keys(p, schema, table).await,
|
||||
PoolKind::Sqlite(p) => db::sqlite::list_foreign_keys(p, schema, table).await,
|
||||
_ => Ok(vec![]),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn list_triggers_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
schema: &str,
|
||||
table: &str,
|
||||
) -> Result<Vec<db::TriggerInfo>, String> {
|
||||
let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?;
|
||||
|
||||
{
|
||||
let connections = state.connections.lock().await;
|
||||
if let Some(client) = extract_sqlserver(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
return db::sqlserver::list_triggers(&mut client, schema, table).await;
|
||||
}
|
||||
if let Some(client) = extract_oracle(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
let client = client.lock().await;
|
||||
return db::oracle_driver::list_triggers(&*client, schema, table).await;
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.lock().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
PoolKind::Mysql(p, _) => db::mysql::list_triggers(p, schema, table).await,
|
||||
PoolKind::Postgres(p) => db::postgres::list_triggers(p, schema, table).await,
|
||||
PoolKind::Sqlite(p) => db::sqlite::list_triggers(p, schema, table).await,
|
||||
_ => Ok(vec![]),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_table_ddl_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
schema: &str,
|
||||
table: &str,
|
||||
) -> Result<String, String> {
|
||||
let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?;
|
||||
|
||||
{
|
||||
let connections = state.connections.lock().await;
|
||||
if let Some(con) = extract_duckdb(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
let tbl = table.replace('\'', "''");
|
||||
let con = con.lock().map_err(|e| e.to_string())?;
|
||||
let mut stmt = con.prepare(&format!("SELECT sql FROM duckdb_tables() WHERE table_name = '{tbl}'"))
|
||||
.map_err(|e| e.to_string())?;
|
||||
let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
|
||||
if let Some(row) = rows.next().map_err(|e| e.to_string())? {
|
||||
return row.get::<_, String>(0).map_err(|e| e.to_string());
|
||||
}
|
||||
return Err("Table not found".to_string());
|
||||
}
|
||||
if let Some(client) = extract_clickhouse(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
let result = db::clickhouse_driver::execute_query(&client, database, &format!("SHOW CREATE TABLE `{table}`")).await?;
|
||||
return result.rows.first()
|
||||
.and_then(|r| r.first())
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or_else(|| "Table not found".to_string());
|
||||
}
|
||||
if let Some(client) = extract_sqlserver(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
return build_sqlserver_ddl(&mut client, schema, table).await;
|
||||
}
|
||||
if let Some(client) = extract_oracle(&connections, &pool_key) {
|
||||
drop(connections);
|
||||
let client = client.lock().await;
|
||||
return build_oracle_ddl(&*client, schema, table).await;
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.lock().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
PoolKind::Mysql(p, _) => mysql_ddl(p, table).await,
|
||||
PoolKind::Postgres(p) => pg_ddl(p, schema, table).await,
|
||||
PoolKind::Sqlite(p) => sqlite_ddl(p, table).await,
|
||||
_ => Err("DDL not supported for this database type".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn mysql_ddl(pool: &sqlx::mysql::MySqlPool, table: &str) -> Result<String, String> {
|
||||
use sqlx::Row;
|
||||
let sql = format!("SHOW CREATE TABLE `{}`", table.replace('`', "``"));
|
||||
let row: sqlx::mysql::MySqlRow = sqlx::raw_sql(&sql)
|
||||
.fetch_one(pool).await.map_err(|e| e.to_string())?;
|
||||
row.try_get::<String, _>(1)
|
||||
.or_else(|_| row.try_get::<Vec<u8>, _>(1).map(|b| String::from_utf8_lossy(&b).to_string()))
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub async fn sqlite_ddl(pool: &sqlx::sqlite::SqlitePool, table: &str) -> Result<String, String> {
|
||||
use sqlx::Row;
|
||||
let row: sqlx::sqlite::SqliteRow = sqlx::query("SELECT sql FROM sqlite_master WHERE type='table' AND name=?")
|
||||
.bind(table)
|
||||
.fetch_one(pool).await.map_err(|e| e.to_string())?;
|
||||
row.try_get::<String, _>(0).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub async fn pg_ddl(pool: &sqlx::postgres::PgPool, schema: &str, table: &str) -> Result<String, String> {
|
||||
let columns = db::postgres::get_columns(pool, schema, table).await?;
|
||||
let indexes = db::postgres::list_indexes(pool, schema, table).await?;
|
||||
let fkeys = db::postgres::list_foreign_keys(pool, schema, table).await?;
|
||||
|
||||
let mut ddl = format!("CREATE TABLE \"{schema}\".\"{table}\" (\n");
|
||||
let col_lines: Vec<String> = columns.iter().map(|c| {
|
||||
let mut line = format!(" \"{}\" {}", c.name, c.data_type);
|
||||
if !c.is_nullable { line.push_str(" NOT NULL"); }
|
||||
if let Some(ref def) = c.column_default { line.push_str(&format!(" DEFAULT {def}")); }
|
||||
line
|
||||
}).collect();
|
||||
ddl.push_str(&col_lines.join(",\n"));
|
||||
|
||||
let pks: Vec<&str> = columns.iter().filter(|c| c.is_primary_key).map(|c| c.name.as_str()).collect();
|
||||
if !pks.is_empty() {
|
||||
ddl.push_str(&format!(",\n PRIMARY KEY ({})", pks.iter().map(|k| format!("\"{k}\"")).collect::<Vec<_>>().join(", ")));
|
||||
}
|
||||
for fk in &fkeys {
|
||||
ddl.push_str(&format!(",\n CONSTRAINT \"{}\" FOREIGN KEY (\"{}\") REFERENCES \"{}\"(\"{}\")", fk.name, fk.column, fk.ref_table, fk.ref_column));
|
||||
}
|
||||
ddl.push_str("\n);\n");
|
||||
|
||||
for idx in &indexes {
|
||||
if idx.is_primary { continue; }
|
||||
let unique = if idx.is_unique { "UNIQUE " } else { "" };
|
||||
let cols = idx.columns.iter().map(|c| format!("\"{c}\"")).collect::<Vec<_>>().join(", ");
|
||||
let using = idx.index_type.as_deref().map(|t| format!(" USING {t}")).unwrap_or_default();
|
||||
let include = idx.included_columns.as_deref().filter(|c| !c.is_empty()).map(|cols| format!(" INCLUDE ({})", cols.iter().map(|c| format!("\"{c}\"")).collect::<Vec<_>>().join(", "))).unwrap_or_default();
|
||||
let filter = idx.filter.as_deref().map(|f| format!(" WHERE {f}")).unwrap_or_default();
|
||||
ddl.push_str(&format!("\nCREATE {unique}INDEX \"{}\" ON \"{schema}\".\"{table}\"{using} ({cols}){include}{filter};", idx.name));
|
||||
if let Some(ref c) = idx.comment {
|
||||
ddl.push_str(&format!("\nCOMMENT ON INDEX \"{schema}\".\"{}\" IS '{}';", idx.name, c.replace('\'', "''")));
|
||||
}
|
||||
}
|
||||
Ok(ddl)
|
||||
}
|
||||
|
||||
pub async fn build_sqlserver_ddl(client: &mut db::sqlserver::SqlServerClient, schema: &str, table: &str) -> Result<String, String> {
|
||||
let columns = db::sqlserver::get_columns(client, schema, table).await?;
|
||||
let indexes = db::sqlserver::list_indexes(client, schema, table).await?;
|
||||
let fkeys = db::sqlserver::list_foreign_keys(client, schema, table).await?;
|
||||
|
||||
let mut ddl = format!("CREATE TABLE [{schema}].[{table}] (\n");
|
||||
let col_lines: Vec<String> = columns.iter().map(|c| {
|
||||
let mut line = format!(" [{}] {}", c.name, c.data_type);
|
||||
if !c.is_nullable { line.push_str(" NOT NULL"); }
|
||||
if let Some(ref def) = c.column_default { line.push_str(&format!(" DEFAULT {def}")); }
|
||||
line
|
||||
}).collect();
|
||||
ddl.push_str(&col_lines.join(",\n"));
|
||||
|
||||
let pks: Vec<&str> = columns.iter().filter(|c| c.is_primary_key).map(|c| c.name.as_str()).collect();
|
||||
if !pks.is_empty() {
|
||||
ddl.push_str(&format!(",\n PRIMARY KEY ({})", pks.iter().map(|k| format!("[{k}]")).collect::<Vec<_>>().join(", ")));
|
||||
}
|
||||
for fk in &fkeys {
|
||||
ddl.push_str(&format!(",\n CONSTRAINT [{}] FOREIGN KEY ([{}]) REFERENCES [{}]([{}])", fk.name, fk.column, fk.ref_table, fk.ref_column));
|
||||
}
|
||||
ddl.push_str("\n);\n");
|
||||
|
||||
for idx in &indexes {
|
||||
if idx.is_primary { continue; }
|
||||
let unique = if idx.is_unique { "UNIQUE " } else { "" };
|
||||
let idx_type = idx.index_type.as_deref().map(|t| format!("{t} ")).unwrap_or_default();
|
||||
let cols = idx.columns.iter().map(|c| format!("[{c}]")).collect::<Vec<_>>().join(", ");
|
||||
let include = idx.included_columns.as_deref().filter(|c| !c.is_empty()).map(|cols| format!(" INCLUDE ({})", cols.iter().map(|c| format!("[{c}]")).collect::<Vec<_>>().join(", "))).unwrap_or_default();
|
||||
let filter = idx.filter.as_deref().map(|f| format!(" WHERE {f}")).unwrap_or_default();
|
||||
ddl.push_str(&format!("\nCREATE {unique}{idx_type}INDEX [{}] ON [{schema}].[{table}] ({cols}){include}{filter};", idx.name));
|
||||
}
|
||||
Ok(ddl)
|
||||
}
|
||||
|
||||
pub async fn build_oracle_ddl(client: &db::oracle_driver::OracleClient, schema: &str, table: &str) -> Result<String, String> {
|
||||
let columns = db::oracle_driver::get_columns(client, schema, table).await?;
|
||||
let indexes = db::oracle_driver::list_indexes(client, schema, table).await?;
|
||||
let fkeys = db::oracle_driver::list_foreign_keys(client, schema, table).await?;
|
||||
|
||||
let mut ddl = format!("CREATE TABLE \"{schema}\".\"{table}\" (\n");
|
||||
let col_lines: Vec<String> = columns.iter().map(|c| {
|
||||
let mut line = format!(" \"{}\" {}", c.name, c.data_type);
|
||||
if !c.is_nullable { line.push_str(" NOT NULL"); }
|
||||
if let Some(ref def) = c.column_default { line.push_str(&format!(" DEFAULT {def}")); }
|
||||
line
|
||||
}).collect();
|
||||
ddl.push_str(&col_lines.join(",\n"));
|
||||
|
||||
let pks: Vec<&str> = columns.iter().filter(|c| c.is_primary_key).map(|c| c.name.as_str()).collect();
|
||||
if !pks.is_empty() {
|
||||
ddl.push_str(&format!(",\n PRIMARY KEY ({})", pks.iter().map(|k| format!("\"{k}\"")).collect::<Vec<_>>().join(", ")));
|
||||
}
|
||||
for fk in &fkeys {
|
||||
ddl.push_str(&format!(",\n CONSTRAINT \"{}\" FOREIGN KEY (\"{}\") REFERENCES \"{}\"(\"{}\")", fk.name, fk.column, fk.ref_table, fk.ref_column));
|
||||
}
|
||||
ddl.push_str("\n);\n");
|
||||
|
||||
for idx in &indexes {
|
||||
if idx.is_primary { continue; }
|
||||
let unique = if idx.is_unique { "UNIQUE " } else { "" };
|
||||
let cols = idx.columns.iter().map(|c| format!("\"{c}\"")).collect::<Vec<_>>().join(", ");
|
||||
ddl.push_str(&format!("\nCREATE {unique}INDEX \"{}\" ON \"{schema}\".\"{table}\" ({cols});", idx.name));
|
||||
}
|
||||
Ok(ddl)
|
||||
}
|
||||
@@ -0,0 +1,415 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct SqlFileRequest {
|
||||
pub execution_id: String,
|
||||
pub connection_id: String,
|
||||
pub database: String,
|
||||
pub file_path: String,
|
||||
pub continue_on_error: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct SqlFilePreview {
|
||||
pub file_name: String,
|
||||
pub file_path: String,
|
||||
pub size_bytes: u64,
|
||||
pub preview: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum SqlFileStatus {
|
||||
Started,
|
||||
Running,
|
||||
StatementDone,
|
||||
StatementFailed,
|
||||
Done,
|
||||
Error,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct SqlFileProgress {
|
||||
pub execution_id: String,
|
||||
pub status: SqlFileStatus,
|
||||
pub statement_index: usize,
|
||||
pub success_count: usize,
|
||||
pub failure_count: usize,
|
||||
pub affected_rows: u64,
|
||||
pub elapsed_ms: u128,
|
||||
pub statement_summary: String,
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct SqlStatementSplitter {
|
||||
buffer: String,
|
||||
in_single_quote: bool,
|
||||
in_double_quote: bool,
|
||||
in_backtick: bool,
|
||||
in_line_comment: bool,
|
||||
in_block_comment: bool,
|
||||
dollar_quote_tag: Option<String>,
|
||||
previous: Option<char>,
|
||||
}
|
||||
|
||||
impl SqlStatementSplitter {
|
||||
pub fn push_chunk(&mut self, chunk: &str) -> Vec<String> {
|
||||
let mut statements = Vec::new();
|
||||
let chars = chunk.chars().collect::<Vec<_>>();
|
||||
let mut i = 0;
|
||||
|
||||
while i < chars.len() {
|
||||
if let Some(tag) = &self.dollar_quote_tag {
|
||||
let tag_chars = tag.chars().collect::<Vec<_>>();
|
||||
if starts_with_chars(&chars, i, &tag_chars) {
|
||||
for tag_ch in &tag_chars {
|
||||
self.buffer.push(*tag_ch);
|
||||
self.previous = Some(*tag_ch);
|
||||
}
|
||||
i += tag_chars.len();
|
||||
self.dollar_quote_tag = None;
|
||||
continue;
|
||||
}
|
||||
|
||||
let ch = chars[i];
|
||||
self.buffer.push(ch);
|
||||
self.previous = Some(ch);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let ch = chars[i];
|
||||
let next = chars.get(i + 1).copied();
|
||||
|
||||
if self.in_line_comment {
|
||||
self.buffer.push(ch);
|
||||
if ch == '\n' {
|
||||
self.in_line_comment = false;
|
||||
}
|
||||
self.previous = Some(ch);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if self.in_block_comment {
|
||||
self.buffer.push(ch);
|
||||
if self.previous == Some('*') && ch == '/' {
|
||||
self.in_block_comment = false;
|
||||
}
|
||||
self.previous = Some(ch);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if !self.in_single_quote && !self.in_double_quote && !self.in_backtick {
|
||||
if self.previous == Some('-') && ch == '-' {
|
||||
self.in_line_comment = true;
|
||||
self.buffer.push(ch);
|
||||
self.previous = Some(ch);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if self.previous == Some('/') && ch == '*' {
|
||||
self.in_block_comment = true;
|
||||
self.buffer.push(ch);
|
||||
self.previous = Some(ch);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if ch == '-' && next == Some('-') {
|
||||
self.in_line_comment = true;
|
||||
self.buffer.push(ch);
|
||||
self.previous = Some(ch);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if ch == '/' && next == Some('*') {
|
||||
self.in_block_comment = true;
|
||||
self.buffer.push(ch);
|
||||
self.previous = Some(ch);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if let Some(tag) = dollar_quote_tag_at(&chars, i) {
|
||||
for tag_ch in tag.chars() {
|
||||
self.buffer.push(tag_ch);
|
||||
self.previous = Some(tag_ch);
|
||||
}
|
||||
i += tag.chars().count();
|
||||
self.dollar_quote_tag = Some(tag);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
match ch {
|
||||
'\'' if !self.in_double_quote
|
||||
&& !self.in_backtick
|
||||
&& self.previous != Some('\\') =>
|
||||
{
|
||||
self.in_single_quote = !self.in_single_quote;
|
||||
self.buffer.push(ch);
|
||||
}
|
||||
'"' if !self.in_single_quote
|
||||
&& !self.in_backtick
|
||||
&& self.previous != Some('\\') =>
|
||||
{
|
||||
self.in_double_quote = !self.in_double_quote;
|
||||
self.buffer.push(ch);
|
||||
}
|
||||
'`' if !self.in_single_quote && !self.in_double_quote => {
|
||||
self.in_backtick = !self.in_backtick;
|
||||
self.buffer.push(ch);
|
||||
}
|
||||
';' if !self.in_single_quote && !self.in_double_quote && !self.in_backtick => {
|
||||
self.push_current_statement(&mut statements);
|
||||
}
|
||||
_ => self.buffer.push(ch),
|
||||
}
|
||||
|
||||
self.previous = Some(ch);
|
||||
i += 1;
|
||||
}
|
||||
|
||||
statements
|
||||
}
|
||||
|
||||
pub fn finish(mut self) -> Vec<String> {
|
||||
let mut statements = Vec::new();
|
||||
self.push_current_statement(&mut statements);
|
||||
statements
|
||||
}
|
||||
|
||||
fn push_current_statement(&mut self, statements: &mut Vec<String>) {
|
||||
let statement = self.buffer.trim();
|
||||
if has_executable_sql(statement) {
|
||||
statements.push(statement.to_string());
|
||||
}
|
||||
self.buffer.clear();
|
||||
self.previous = None;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn split_sql_statements(sql: &str) -> Vec<String> {
|
||||
let mut splitter = SqlStatementSplitter::default();
|
||||
let mut statements = splitter.push_chunk(sql);
|
||||
statements.extend(splitter.finish());
|
||||
statements
|
||||
}
|
||||
|
||||
pub fn statement_summary(statement: &str) -> String {
|
||||
const MAX_LEN: usize = 120;
|
||||
|
||||
let collapsed = statement.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
if collapsed.chars().count() <= MAX_LEN {
|
||||
return collapsed;
|
||||
}
|
||||
|
||||
collapsed.chars().take(MAX_LEN).collect()
|
||||
}
|
||||
|
||||
fn starts_with_chars(chars: &[char], start: usize, needle: &[char]) -> bool {
|
||||
start + needle.len() <= chars.len() && chars[start..start + needle.len()] == *needle
|
||||
}
|
||||
|
||||
fn dollar_quote_tag_at(chars: &[char], start: usize) -> Option<String> {
|
||||
if chars.get(start) != Some(&'$') {
|
||||
return None;
|
||||
}
|
||||
|
||||
match chars.get(start + 1) {
|
||||
Some('$') => return Some("$$".to_string()),
|
||||
Some(ch) if ch.is_ascii_alphabetic() || *ch == '_' => {}
|
||||
_ => return None,
|
||||
}
|
||||
|
||||
let mut end = start + 2;
|
||||
while let Some(ch) = chars.get(end) {
|
||||
if *ch == '$' {
|
||||
return Some(chars[start..=end].iter().collect());
|
||||
}
|
||||
if !ch.is_ascii_alphanumeric() && *ch != '_' {
|
||||
return None;
|
||||
}
|
||||
end += 1;
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn has_executable_sql(statement: &str) -> bool {
|
||||
let chars = statement.chars().collect::<Vec<_>>();
|
||||
let mut in_line_comment = false;
|
||||
let mut in_block_comment = false;
|
||||
let mut previous = None;
|
||||
let mut i = 0;
|
||||
|
||||
while i < chars.len() {
|
||||
let ch = chars[i];
|
||||
let next = chars.get(i + 1).copied();
|
||||
|
||||
if in_line_comment {
|
||||
if ch == '\n' {
|
||||
in_line_comment = false;
|
||||
}
|
||||
previous = Some(ch);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if in_block_comment {
|
||||
if previous == Some('*') && ch == '/' {
|
||||
in_block_comment = false;
|
||||
}
|
||||
previous = Some(ch);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ch == '-' && next == Some('-') {
|
||||
in_line_comment = true;
|
||||
previous = Some(ch);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ch == '/' && next == Some('*') {
|
||||
if is_mysql_executable_comment_start(&chars, i) {
|
||||
return true;
|
||||
}
|
||||
in_block_comment = true;
|
||||
previous = Some(ch);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if !ch.is_whitespace() {
|
||||
return true;
|
||||
}
|
||||
|
||||
previous = Some(ch);
|
||||
i += 1;
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
fn is_mysql_executable_comment_start(chars: &[char], start: usize) -> bool {
|
||||
chars.get(start) == Some(&'/')
|
||||
&& chars.get(start + 1) == Some(&'*')
|
||||
&& (chars.get(start + 2) == Some(&'!')
|
||||
|| (chars.get(start + 2) == Some(&'M') && chars.get(start + 3) == Some(&'!')))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn split_sql_script(sql: &str) -> Result<Vec<String>, String> {
|
||||
Ok(split_sql_statements(sql))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{split_sql_script, SqlStatementSplitter};
|
||||
|
||||
#[test]
|
||||
fn splits_semicolon_delimited_statements() {
|
||||
assert_eq!(
|
||||
split_sql_script("CREATE TABLE a(id int); INSERT INTO a VALUES (1);").unwrap(),
|
||||
vec!["CREATE TABLE a(id int)", "INSERT INTO a VALUES (1)"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_semicolons_inside_quotes_and_comments() {
|
||||
let sql = "\
|
||||
INSERT INTO logs VALUES ('a;b', \"c;d\", `weird;name`);\n\
|
||||
-- comment ; ignored\n\
|
||||
/* block ; ignored */\n\
|
||||
SELECT 1;";
|
||||
assert_eq!(
|
||||
split_sql_script(sql).unwrap(),
|
||||
vec![
|
||||
"INSERT INTO logs VALUES ('a;b', \"c;d\", `weird;name`)",
|
||||
"-- comment ; ignored\n/* block ; ignored */\nSELECT 1",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn emits_trailing_statement_without_semicolon() {
|
||||
assert_eq!(
|
||||
split_sql_script("CREATE TABLE a(id int);\nINSERT INTO a VALUES (1)").unwrap(),
|
||||
vec!["CREATE TABLE a(id int)", "INSERT INTO a VALUES (1)"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn line_comment_openers_can_span_chunks() {
|
||||
let mut splitter = SqlStatementSplitter::default();
|
||||
|
||||
assert_eq!(splitter.push_chunk("SELECT 1; -"), vec!["SELECT 1"]);
|
||||
assert_eq!(
|
||||
splitter.push_chunk("- comment ; ignored\nSELECT 2;"),
|
||||
vec!["-- comment ; ignored\nSELECT 2"]
|
||||
);
|
||||
assert_eq!(splitter.finish(), Vec::<String>::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn block_comment_openers_can_span_chunks() {
|
||||
let mut splitter = SqlStatementSplitter::default();
|
||||
|
||||
assert_eq!(splitter.push_chunk("SELECT 1; /"), vec!["SELECT 1"]);
|
||||
assert_eq!(
|
||||
splitter.push_chunk("* comment ; ignored */\nSELECT 2;"),
|
||||
vec!["/* comment ; ignored */\nSELECT 2"]
|
||||
);
|
||||
assert_eq!(splitter.finish(), Vec::<String>::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skips_comment_only_tail_after_statement() {
|
||||
assert_eq!(
|
||||
split_sql_script("CREATE TABLE a(id int); -- done\n/* no more sql */").unwrap(),
|
||||
vec!["CREATE TABLE a(id int)"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_postgres_dollar_quoted_function_body_together() {
|
||||
let sql = "\
|
||||
CREATE FUNCTION bump_counter()\n\
|
||||
RETURNS trigger AS $$\n\
|
||||
BEGIN\n\
|
||||
PERFORM 1;\n\
|
||||
RETURN NEW;\n\
|
||||
END;\n\
|
||||
$$ LANGUAGE plpgsql;\n\
|
||||
SELECT 1;";
|
||||
|
||||
assert_eq!(
|
||||
split_sql_script(sql).unwrap(),
|
||||
vec![
|
||||
"CREATE FUNCTION bump_counter()\nRETURNS trigger AS $$\nBEGIN\nPERFORM 1;\nRETURN NEW;\nEND;\n$$ LANGUAGE plpgsql",
|
||||
"SELECT 1",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_mysql_executable_comments_as_statements() {
|
||||
assert_eq!(
|
||||
split_sql_script(
|
||||
"/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;\nSELECT 1;",
|
||||
)
|
||||
.unwrap(),
|
||||
vec![
|
||||
"/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */",
|
||||
"SELECT 1",
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,717 @@
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
|
||||
use calamine::{open_workbook_auto, Data, Reader};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::models::connection::DatabaseType;
|
||||
use crate::transfer::{execute_on_pool, generate_insert, qualified_table};
|
||||
use crate::connection::AppState;
|
||||
|
||||
pub const DEFAULT_PREVIEW_LIMIT: usize = 50;
|
||||
pub const DEFAULT_BATCH_SIZE: usize = 500;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ParsedImportFile {
|
||||
pub columns: Vec<String>,
|
||||
pub rows: Vec<Vec<serde_json::Value>>,
|
||||
pub total_rows: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ImportSqlBatch {
|
||||
pub sql: String,
|
||||
pub row_count: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TableImportColumnMapping {
|
||||
pub source_column: String,
|
||||
pub target_column: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum TableImportMode {
|
||||
Append,
|
||||
Truncate,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TableImportRequest {
|
||||
pub import_id: String,
|
||||
pub connection_id: String,
|
||||
pub database: String,
|
||||
pub schema: String,
|
||||
pub table: String,
|
||||
pub file_path: String,
|
||||
pub mappings: Vec<TableImportColumnMapping>,
|
||||
pub mode: TableImportMode,
|
||||
pub batch_size: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TableImportPreview {
|
||||
pub file_name: String,
|
||||
pub file_path: String,
|
||||
pub file_type: String,
|
||||
pub size_bytes: u64,
|
||||
pub columns: Vec<String>,
|
||||
pub rows: Vec<Vec<serde_json::Value>>,
|
||||
pub total_rows: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TableImportSummary {
|
||||
pub import_id: String,
|
||||
pub rows_imported: usize,
|
||||
pub total_rows: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TableImportProgress {
|
||||
pub import_id: String,
|
||||
pub status: TableImportStatus,
|
||||
pub rows_imported: usize,
|
||||
pub total_rows: usize,
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum TableImportStatus {
|
||||
Running,
|
||||
Done,
|
||||
Error,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ImportFileKind {
|
||||
Csv,
|
||||
Tsv,
|
||||
Json,
|
||||
Xlsx,
|
||||
}
|
||||
|
||||
impl ImportFileKind {
|
||||
pub fn label(self) -> &'static str {
|
||||
match self {
|
||||
ImportFileKind::Csv => "csv",
|
||||
ImportFileKind::Tsv => "tsv",
|
||||
ImportFileKind::Json => "json",
|
||||
ImportFileKind::Xlsx => "xlsx",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn import_file_kind(path: &str) -> Result<ImportFileKind, String> {
|
||||
let lower = path.to_lowercase();
|
||||
if lower.ends_with(".csv") {
|
||||
Ok(ImportFileKind::Csv)
|
||||
} else if lower.ends_with(".tsv") {
|
||||
Ok(ImportFileKind::Tsv)
|
||||
} else if lower.ends_with(".json") {
|
||||
Ok(ImportFileKind::Json)
|
||||
} else if lower.ends_with(".xlsx") || lower.ends_with(".xlsm") || lower.ends_with(".xls") {
|
||||
Ok(ImportFileKind::Xlsx)
|
||||
} else {
|
||||
Err("Unsupported import file type".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn normalize_header(value: &str, index: usize) -> String {
|
||||
let trimmed = value.trim();
|
||||
if trimmed.is_empty() {
|
||||
format!("column_{}", index + 1)
|
||||
} else {
|
||||
trimmed.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn csv_value(value: &str) -> serde_json::Value {
|
||||
if value.is_empty() {
|
||||
serde_json::Value::Null
|
||||
} else {
|
||||
serde_json::Value::String(value.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_delimited_bytes(
|
||||
bytes: &[u8],
|
||||
delimiter: u8,
|
||||
preview_limit: usize,
|
||||
) -> Result<ParsedImportFile, String> {
|
||||
let mut reader = csv::ReaderBuilder::new()
|
||||
.delimiter(delimiter)
|
||||
.flexible(true)
|
||||
.from_reader(bytes);
|
||||
let columns = reader
|
||||
.headers()
|
||||
.map_err(|e| e.to_string())?
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, header)| normalize_header(header, index))
|
||||
.collect::<Vec<_>>();
|
||||
if columns.is_empty() {
|
||||
return Err("Import file has no columns".to_string());
|
||||
}
|
||||
|
||||
let mut rows = Vec::new();
|
||||
let mut total_rows = 0;
|
||||
for record in reader.records() {
|
||||
let record = record.map_err(|e| e.to_string())?;
|
||||
total_rows += 1;
|
||||
if rows.len() >= preview_limit {
|
||||
continue;
|
||||
}
|
||||
let mut row = Vec::with_capacity(columns.len());
|
||||
for index in 0..columns.len() {
|
||||
row.push(
|
||||
record
|
||||
.get(index)
|
||||
.map(csv_value)
|
||||
.unwrap_or(serde_json::Value::Null),
|
||||
);
|
||||
}
|
||||
rows.push(row);
|
||||
}
|
||||
|
||||
Ok(ParsedImportFile {
|
||||
columns,
|
||||
rows,
|
||||
total_rows,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn parse_csv_bytes(bytes: &[u8], preview_limit: usize) -> Result<ParsedImportFile, String> {
|
||||
parse_delimited_bytes(bytes, b',', preview_limit)
|
||||
}
|
||||
|
||||
pub fn parse_json_bytes(bytes: &[u8], preview_limit: usize) -> Result<ParsedImportFile, String> {
|
||||
let value: serde_json::Value = serde_json::from_slice(bytes).map_err(|e| e.to_string())?;
|
||||
let items = match value {
|
||||
serde_json::Value::Array(items) => items,
|
||||
serde_json::Value::Object(_) => vec![value],
|
||||
_ => return Err("JSON import must be an object or an array".to_string()),
|
||||
};
|
||||
if items.is_empty() {
|
||||
return Err("Import file has no rows".to_string());
|
||||
}
|
||||
|
||||
if items.iter().all(|item| item.is_object()) {
|
||||
let mut columns = Vec::new();
|
||||
for item in &items {
|
||||
if let Some(obj) = item.as_object() {
|
||||
for key in obj.keys() {
|
||||
if !columns.contains(key) {
|
||||
columns.push(key.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if columns.is_empty() {
|
||||
return Err("Import file has no columns".to_string());
|
||||
}
|
||||
let rows = items
|
||||
.iter()
|
||||
.take(preview_limit)
|
||||
.map(|item| {
|
||||
let obj = item.as_object().expect("checked object JSON row");
|
||||
columns
|
||||
.iter()
|
||||
.map(|column| obj.get(column).cloned().unwrap_or(serde_json::Value::Null))
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
return Ok(ParsedImportFile {
|
||||
columns,
|
||||
rows,
|
||||
total_rows: items.len(),
|
||||
});
|
||||
}
|
||||
|
||||
if items.iter().all(|item| item.is_array()) {
|
||||
let max_cols = items
|
||||
.iter()
|
||||
.filter_map(|item| item.as_array().map(|row| row.len()))
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
if max_cols == 0 {
|
||||
return Err("Import file has no columns".to_string());
|
||||
}
|
||||
let columns = (0..max_cols)
|
||||
.map(|index| format!("column_{}", index + 1))
|
||||
.collect::<Vec<_>>();
|
||||
let rows = items
|
||||
.iter()
|
||||
.take(preview_limit)
|
||||
.map(|item| {
|
||||
let arr = item.as_array().expect("checked array JSON row");
|
||||
(0..max_cols)
|
||||
.map(|index| arr.get(index).cloned().unwrap_or(serde_json::Value::Null))
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
return Ok(ParsedImportFile {
|
||||
columns,
|
||||
rows,
|
||||
total_rows: items.len(),
|
||||
});
|
||||
}
|
||||
|
||||
Err("JSON rows must all be objects or all be arrays".to_string())
|
||||
}
|
||||
|
||||
pub fn xlsx_cell_value(cell: &Data) -> serde_json::Value {
|
||||
match cell {
|
||||
Data::Empty => serde_json::Value::Null,
|
||||
Data::String(s) => csv_value(s),
|
||||
Data::Float(n) => serde_json::Number::from_f64(*n)
|
||||
.map(serde_json::Value::Number)
|
||||
.unwrap_or(serde_json::Value::Null),
|
||||
Data::Int(n) => serde_json::Value::Number((*n).into()),
|
||||
Data::Bool(v) => serde_json::Value::Bool(*v),
|
||||
Data::DateTime(v) => serde_json::Value::String(v.to_string()),
|
||||
Data::DateTimeIso(v) => serde_json::Value::String(v.clone()),
|
||||
Data::DurationIso(v) => serde_json::Value::String(v.clone()),
|
||||
Data::Error(v) => serde_json::Value::String(v.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn xlsx_cell_label(cell: &Data) -> String {
|
||||
match cell {
|
||||
Data::Empty => String::new(),
|
||||
Data::String(s) => s.clone(),
|
||||
Data::Float(n) => n.to_string(),
|
||||
Data::Int(n) => n.to_string(),
|
||||
Data::Bool(v) => v.to_string(),
|
||||
Data::DateTime(v) => v.to_string(),
|
||||
Data::DateTimeIso(v) => v.clone(),
|
||||
Data::DurationIso(v) => v.clone(),
|
||||
Data::Error(v) => v.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_xlsx_file(path: &str, preview_limit: usize) -> Result<ParsedImportFile, String> {
|
||||
let mut workbook = open_workbook_auto(path).map_err(|e| e.to_string())?;
|
||||
let sheet_name = workbook
|
||||
.sheet_names()
|
||||
.first()
|
||||
.cloned()
|
||||
.ok_or_else(|| "Workbook has no sheets".to_string())?;
|
||||
let range = workbook
|
||||
.worksheet_range(&sheet_name)
|
||||
.map_err(|e| e.to_string())?;
|
||||
let mut rows_iter = range.rows();
|
||||
let header = rows_iter
|
||||
.next()
|
||||
.ok_or_else(|| "Import file has no rows".to_string())?;
|
||||
let columns = header
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, cell)| normalize_header(&xlsx_cell_label(cell), index))
|
||||
.collect::<Vec<_>>();
|
||||
if columns.is_empty() {
|
||||
return Err("Import file has no columns".to_string());
|
||||
}
|
||||
|
||||
let mut rows = Vec::new();
|
||||
let mut total_rows = 0;
|
||||
for source_row in rows_iter {
|
||||
total_rows += 1;
|
||||
if rows.len() >= preview_limit {
|
||||
continue;
|
||||
}
|
||||
let mut row = Vec::with_capacity(columns.len());
|
||||
for index in 0..columns.len() {
|
||||
row.push(
|
||||
source_row
|
||||
.get(index)
|
||||
.map(xlsx_cell_value)
|
||||
.unwrap_or(serde_json::Value::Null),
|
||||
);
|
||||
}
|
||||
rows.push(row);
|
||||
}
|
||||
|
||||
Ok(ParsedImportFile {
|
||||
columns,
|
||||
rows,
|
||||
total_rows,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn parse_import_file(path: &str, preview_limit: usize) -> Result<ParsedImportFile, String> {
|
||||
match import_file_kind(path)? {
|
||||
ImportFileKind::Csv => {
|
||||
let bytes = std::fs::read(path).map_err(|e| e.to_string())?;
|
||||
parse_csv_bytes(&bytes, preview_limit)
|
||||
}
|
||||
ImportFileKind::Tsv => {
|
||||
let bytes = std::fs::read(path).map_err(|e| e.to_string())?;
|
||||
parse_delimited_bytes(&bytes, b'\t', preview_limit)
|
||||
}
|
||||
ImportFileKind::Json => {
|
||||
let bytes = std::fs::read(path).map_err(|e| e.to_string())?;
|
||||
parse_json_bytes(&bytes, preview_limit)
|
||||
}
|
||||
ImportFileKind::Xlsx => parse_xlsx_file(path, preview_limit),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn mapping_indexes(
|
||||
data: &ParsedImportFile,
|
||||
mappings: &[TableImportColumnMapping],
|
||||
) -> Result<Vec<(usize, String)>, String> {
|
||||
if mappings.is_empty() {
|
||||
return Err("No columns mapped for import".to_string());
|
||||
}
|
||||
let mut mapped = Vec::new();
|
||||
let mut target_seen = HashSet::new();
|
||||
for mapping in mappings {
|
||||
let source_index = data
|
||||
.columns
|
||||
.iter()
|
||||
.position(|column| column == &mapping.source_column)
|
||||
.ok_or_else(|| format!("Source column not found: {}", mapping.source_column))?;
|
||||
if mapping.target_column.trim().is_empty() {
|
||||
return Err("Target column cannot be empty".to_string());
|
||||
}
|
||||
if !target_seen.insert(mapping.target_column.clone()) {
|
||||
return Err(format!(
|
||||
"Target column mapped more than once: {}",
|
||||
mapping.target_column
|
||||
));
|
||||
}
|
||||
mapped.push((source_index, mapping.target_column.clone()));
|
||||
}
|
||||
Ok(mapped)
|
||||
}
|
||||
|
||||
pub fn build_import_insert_batches(
|
||||
data: &ParsedImportFile,
|
||||
mappings: &[TableImportColumnMapping],
|
||||
table: &str,
|
||||
schema: &str,
|
||||
db_type: &DatabaseType,
|
||||
batch_size: usize,
|
||||
) -> Result<Vec<ImportSqlBatch>, String> {
|
||||
let mapped = mapping_indexes(data, mappings)?;
|
||||
let columns = mapped
|
||||
.iter()
|
||||
.map(|(_, target)| target.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let batch_size = batch_size.max(1);
|
||||
let mut batches = Vec::new();
|
||||
|
||||
for chunk in data.rows.chunks(batch_size) {
|
||||
let rows = chunk
|
||||
.iter()
|
||||
.map(|row| {
|
||||
mapped
|
||||
.iter()
|
||||
.map(|(source_index, _)| {
|
||||
row.get(*source_index)
|
||||
.cloned()
|
||||
.unwrap_or(serde_json::Value::Null)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let sql = generate_insert(&columns, &rows, table, schema, db_type);
|
||||
if !sql.trim().is_empty() {
|
||||
batches.push(ImportSqlBatch {
|
||||
sql,
|
||||
row_count: chunk.len(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(batches)
|
||||
}
|
||||
|
||||
pub fn truncate_sql(table: &str, schema: &str, db_type: &DatabaseType) -> String {
|
||||
let full_table = qualified_table(table, schema, db_type);
|
||||
match db_type {
|
||||
DatabaseType::Sqlite => format!("DELETE FROM {full_table}"),
|
||||
_ => format!("TRUNCATE TABLE {full_table}"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn preview_table_import_file_core(file_path: &str) -> Result<TableImportPreview, String> {
|
||||
let kind = import_file_kind(file_path)?;
|
||||
let parsed = parse_import_file(file_path, DEFAULT_PREVIEW_LIMIT)?;
|
||||
let metadata = std::fs::metadata(file_path).map_err(|e| e.to_string())?;
|
||||
let file_name = Path::new(file_path)
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or(file_path)
|
||||
.to_string();
|
||||
|
||||
Ok(TableImportPreview {
|
||||
file_name,
|
||||
file_path: file_path.to_string(),
|
||||
file_type: kind.label().to_string(),
|
||||
size_bytes: metadata.len(),
|
||||
columns: parsed.columns,
|
||||
rows: parsed.rows,
|
||||
total_rows: parsed.total_rows,
|
||||
})
|
||||
}
|
||||
|
||||
/// Core import logic. Returns (rows_imported, total_rows).
|
||||
/// `progress_callback` is invoked for progress updates.
|
||||
pub async fn import_table_file_core<F>(
|
||||
state: &AppState,
|
||||
request: &TableImportRequest,
|
||||
db_type: &DatabaseType,
|
||||
pool_key: &str,
|
||||
is_cancelled: impl Fn(&str) -> std::pin::Pin<Box<dyn std::future::Future<Output = bool> + Send + '_>>,
|
||||
mut progress_callback: F,
|
||||
) -> Result<TableImportSummary, String>
|
||||
where
|
||||
F: FnMut(TableImportProgress),
|
||||
{
|
||||
let batch_size = if request.batch_size == 0 {
|
||||
DEFAULT_BATCH_SIZE
|
||||
} else {
|
||||
request.batch_size
|
||||
};
|
||||
|
||||
let parsed = match parse_import_file(&request.file_path, usize::MAX) {
|
||||
Ok(parsed) => parsed,
|
||||
Err(error) => {
|
||||
progress_callback(TableImportProgress {
|
||||
import_id: request.import_id.clone(),
|
||||
status: TableImportStatus::Error,
|
||||
rows_imported: 0,
|
||||
total_rows: 0,
|
||||
error: Some(error.clone()),
|
||||
});
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
|
||||
let total_rows = parsed.total_rows;
|
||||
progress_callback(TableImportProgress {
|
||||
import_id: request.import_id.clone(),
|
||||
status: TableImportStatus::Running,
|
||||
rows_imported: 0,
|
||||
total_rows,
|
||||
error: None,
|
||||
});
|
||||
|
||||
let batches = match build_import_insert_batches(
|
||||
&parsed,
|
||||
&request.mappings,
|
||||
&request.table,
|
||||
&request.schema,
|
||||
db_type,
|
||||
batch_size,
|
||||
) {
|
||||
Ok(batches) => batches,
|
||||
Err(error) => {
|
||||
progress_callback(TableImportProgress {
|
||||
import_id: request.import_id.clone(),
|
||||
status: TableImportStatus::Error,
|
||||
rows_imported: 0,
|
||||
total_rows,
|
||||
error: Some(error.clone()),
|
||||
});
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
|
||||
if matches!(request.mode, TableImportMode::Truncate) {
|
||||
let sql = truncate_sql(&request.table, &request.schema, db_type);
|
||||
if let Err(error) = execute_on_pool(state, pool_key, &sql).await {
|
||||
progress_callback(TableImportProgress {
|
||||
import_id: request.import_id.clone(),
|
||||
status: TableImportStatus::Error,
|
||||
rows_imported: 0,
|
||||
total_rows,
|
||||
error: Some(error.clone()),
|
||||
});
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
|
||||
let mut rows_imported = 0;
|
||||
for batch in batches {
|
||||
if is_cancelled(&request.import_id).await {
|
||||
progress_callback(TableImportProgress {
|
||||
import_id: request.import_id.clone(),
|
||||
status: TableImportStatus::Cancelled,
|
||||
rows_imported,
|
||||
total_rows,
|
||||
error: None,
|
||||
});
|
||||
return Err("Import cancelled".to_string());
|
||||
}
|
||||
|
||||
if let Err(error) = execute_on_pool(state, pool_key, &batch.sql).await {
|
||||
progress_callback(TableImportProgress {
|
||||
import_id: request.import_id.clone(),
|
||||
status: TableImportStatus::Error,
|
||||
rows_imported,
|
||||
total_rows,
|
||||
error: Some(error.clone()),
|
||||
});
|
||||
return Err(error);
|
||||
}
|
||||
rows_imported = (rows_imported + batch.row_count).min(total_rows);
|
||||
progress_callback(TableImportProgress {
|
||||
import_id: request.import_id.clone(),
|
||||
status: TableImportStatus::Running,
|
||||
rows_imported,
|
||||
total_rows,
|
||||
error: None,
|
||||
});
|
||||
}
|
||||
|
||||
progress_callback(TableImportProgress {
|
||||
import_id: request.import_id.clone(),
|
||||
status: TableImportStatus::Done,
|
||||
rows_imported,
|
||||
total_rows,
|
||||
error: None,
|
||||
});
|
||||
|
||||
Ok(TableImportSummary {
|
||||
import_id: request.import_id.clone(),
|
||||
rows_imported,
|
||||
total_rows,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::models::connection::DatabaseType;
|
||||
|
||||
#[test]
|
||||
fn parses_csv_headers_and_preview_rows() {
|
||||
let parsed = parse_csv_bytes(b"id,name,active\n1,Ada,true\n2,,false\n", 10).unwrap();
|
||||
|
||||
assert_eq!(parsed.columns, vec!["id", "name", "active"]);
|
||||
assert_eq!(parsed.total_rows, 2);
|
||||
assert_eq!(
|
||||
parsed.rows[0],
|
||||
vec![
|
||||
serde_json::Value::String("1".to_string()),
|
||||
serde_json::Value::String("Ada".to_string()),
|
||||
serde_json::Value::String("true".to_string()),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
parsed.rows[1],
|
||||
vec![
|
||||
serde_json::Value::String("2".to_string()),
|
||||
serde_json::Value::Null,
|
||||
serde_json::Value::String("false".to_string()),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_tsv_with_tab_delimiter() {
|
||||
let parsed = parse_delimited_bytes(b"id\tname\n1\tAda\n", b'\t', 10).unwrap();
|
||||
|
||||
assert_eq!(parsed.columns, vec!["id", "name"]);
|
||||
assert_eq!(parsed.total_rows, 1);
|
||||
assert_eq!(
|
||||
parsed.rows[0],
|
||||
vec![
|
||||
serde_json::Value::String("1".to_string()),
|
||||
serde_json::Value::String("Ada".to_string()),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_json_array_objects_with_union_columns() {
|
||||
let parsed =
|
||||
parse_json_bytes(br#"[{"id":1,"name":"Ada"},{"id":2,"active":true}]"#, 10).unwrap();
|
||||
|
||||
assert_eq!(parsed.columns, vec!["id", "name", "active"]);
|
||||
assert_eq!(parsed.total_rows, 2);
|
||||
assert_eq!(
|
||||
parsed.rows[0],
|
||||
vec![
|
||||
serde_json::json!(1),
|
||||
serde_json::json!("Ada"),
|
||||
serde_json::Value::Null,
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
parsed.rows[1],
|
||||
vec![
|
||||
serde_json::json!(2),
|
||||
serde_json::Value::Null,
|
||||
serde_json::json!(true),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_import_insert_batches_from_mapped_columns() {
|
||||
let mappings = vec![
|
||||
TableImportColumnMapping {
|
||||
source_column: "id".to_string(),
|
||||
target_column: "user_id".to_string(),
|
||||
},
|
||||
TableImportColumnMapping {
|
||||
source_column: "name".to_string(),
|
||||
target_column: "display_name".to_string(),
|
||||
},
|
||||
];
|
||||
let data = ParsedImportFile {
|
||||
columns: vec!["id".to_string(), "name".to_string(), "ignored".to_string()],
|
||||
rows: vec![
|
||||
vec![
|
||||
serde_json::json!(1),
|
||||
serde_json::json!("Ada"),
|
||||
serde_json::json!("x"),
|
||||
],
|
||||
vec![
|
||||
serde_json::json!(2),
|
||||
serde_json::json!("O'Hara"),
|
||||
serde_json::json!("y"),
|
||||
],
|
||||
vec![
|
||||
serde_json::json!(3),
|
||||
serde_json::Value::Null,
|
||||
serde_json::json!("z"),
|
||||
],
|
||||
],
|
||||
total_rows: 3,
|
||||
};
|
||||
|
||||
let batches = build_import_insert_batches(
|
||||
&data,
|
||||
&mappings,
|
||||
"users",
|
||||
"public",
|
||||
&DatabaseType::Postgres,
|
||||
2,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(batches, vec![
|
||||
ImportSqlBatch {
|
||||
sql: "INSERT INTO \"public\".\"users\" (\"user_id\", \"display_name\") VALUES\n(1, 'Ada'),\n(2, 'O''Hara')".to_string(),
|
||||
row_count: 2,
|
||||
},
|
||||
ImportSqlBatch {
|
||||
sql: "INSERT INTO \"public\".\"users\" (\"user_id\", \"display_name\") VALUES\n(3, NULL)".to_string(),
|
||||
row_count: 1,
|
||||
},
|
||||
]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,630 @@
|
||||
use std::collections::HashSet;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use crate::connection::{AppState, PoolKind};
|
||||
use crate::db;
|
||||
use crate::models::connection::DatabaseType;
|
||||
|
||||
static CANCELLED: std::sync::LazyLock<RwLock<HashSet<String>>> =
|
||||
std::sync::LazyLock::new(|| RwLock::new(HashSet::new()));
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TransferRequest {
|
||||
pub transfer_id: String,
|
||||
pub source_connection_id: String,
|
||||
pub source_database: String,
|
||||
pub source_schema: String,
|
||||
pub target_connection_id: String,
|
||||
pub target_database: String,
|
||||
pub target_schema: String,
|
||||
pub tables: Vec<String>,
|
||||
pub create_table: bool,
|
||||
pub truncate_before: bool,
|
||||
pub batch_size: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TransferProgress {
|
||||
pub transfer_id: String,
|
||||
pub table: String,
|
||||
pub table_index: usize,
|
||||
pub total_tables: usize,
|
||||
pub rows_transferred: u64,
|
||||
pub total_rows: Option<u64>,
|
||||
pub status: TransferStatus,
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum TransferStatus {
|
||||
Running,
|
||||
TableDone,
|
||||
Done,
|
||||
Error,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
pub fn quote_identifier(name: &str, db_type: &DatabaseType) -> String {
|
||||
match db_type {
|
||||
DatabaseType::Mysql | DatabaseType::ClickHouse | DatabaseType::Doris | DatabaseType::StarRocks => format!("`{}`", name.replace('`', "``")),
|
||||
DatabaseType::SqlServer => format!("[{}]", name.replace(']', "]]")),
|
||||
_ => format!("\"{}\"", name.replace('"', "\"\"")),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn qualified_table(table: &str, schema: &str, db_type: &DatabaseType) -> String {
|
||||
let qt = quote_identifier(table, db_type);
|
||||
if schema.is_empty() {
|
||||
qt
|
||||
} else {
|
||||
format!("{}.{}", quote_identifier(schema, db_type), qt)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn escape_value(val: &serde_json::Value, db_type: &DatabaseType) -> String {
|
||||
match val {
|
||||
serde_json::Value::Null => "NULL".to_string(),
|
||||
serde_json::Value::Bool(b) => match db_type {
|
||||
DatabaseType::Mysql | DatabaseType::Sqlite | DatabaseType::DuckDb | DatabaseType::Doris | DatabaseType::StarRocks => {
|
||||
if *b { "1".to_string() } else { "0".to_string() }
|
||||
}
|
||||
_ => if *b { "TRUE".to_string() } else { "FALSE".to_string() },
|
||||
},
|
||||
serde_json::Value::Number(n) => n.to_string(),
|
||||
serde_json::Value::String(s) => {
|
||||
format!("'{}'", s.replace('\'', "''"))
|
||||
}
|
||||
_ => {
|
||||
let s = val.to_string();
|
||||
format!("'{}'", s.replace('\'', "''"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn map_column_type(source_type: &str, _source_db: &DatabaseType, target_db: &DatabaseType) -> String {
|
||||
let t = source_type.to_lowercase();
|
||||
let base = t.split('(').next().unwrap_or(&t).trim();
|
||||
|
||||
match base {
|
||||
"int" | "integer" | "int4" | "mediumint" => match target_db {
|
||||
DatabaseType::Postgres => "INTEGER".into(),
|
||||
DatabaseType::Mysql => "INT".into(),
|
||||
DatabaseType::SqlServer => "INT".into(),
|
||||
_ => "INTEGER".into(),
|
||||
},
|
||||
"bigint" | "int8" => "BIGINT".into(),
|
||||
"smallint" | "int2" => "SMALLINT".into(),
|
||||
"tinyint" => match target_db {
|
||||
DatabaseType::Postgres => "SMALLINT".into(),
|
||||
_ => "TINYINT".into(),
|
||||
},
|
||||
"serial" | "bigserial" | "smallserial" => match target_db {
|
||||
DatabaseType::Postgres => source_type.to_uppercase(),
|
||||
DatabaseType::Mysql => "BIGINT AUTO_INCREMENT".into(),
|
||||
_ => "INTEGER".into(),
|
||||
},
|
||||
"float" | "float4" | "real" => match target_db {
|
||||
DatabaseType::Postgres => "REAL".into(),
|
||||
_ => "FLOAT".into(),
|
||||
},
|
||||
"double" | "double precision" | "float8" => match target_db {
|
||||
DatabaseType::Postgres => "DOUBLE PRECISION".into(),
|
||||
_ => "DOUBLE".into(),
|
||||
},
|
||||
"decimal" | "numeric" | "number" => {
|
||||
if t.contains('(') {
|
||||
match target_db {
|
||||
DatabaseType::Mysql | DatabaseType::Postgres | DatabaseType::SqlServer | DatabaseType::Oracle => {
|
||||
format!("DECIMAL{}", &t[t.find('(').unwrap()..])
|
||||
}
|
||||
_ => "NUMERIC".into(),
|
||||
}
|
||||
} else {
|
||||
"NUMERIC".into()
|
||||
}
|
||||
}
|
||||
"varchar" | "nvarchar" | "character varying" | "varchar2" => {
|
||||
if t.contains('(') {
|
||||
let len_part = &t[t.find('(').unwrap()..];
|
||||
match target_db {
|
||||
DatabaseType::Postgres => format!("VARCHAR{len_part}"),
|
||||
DatabaseType::Mysql => format!("VARCHAR{len_part}"),
|
||||
DatabaseType::SqlServer => format!("NVARCHAR{len_part}"),
|
||||
_ => format!("VARCHAR{len_part}"),
|
||||
}
|
||||
} else {
|
||||
"VARCHAR(255)".into()
|
||||
}
|
||||
}
|
||||
"char" | "nchar" | "character" => {
|
||||
if t.contains('(') {
|
||||
let len_part = &t[t.find('(').unwrap()..];
|
||||
format!("CHAR{len_part}")
|
||||
} else {
|
||||
"CHAR(1)".into()
|
||||
}
|
||||
}
|
||||
"text" | "longtext" | "mediumtext" | "tinytext" | "clob" | "ntext" => "TEXT".into(),
|
||||
"bool" | "boolean" => match target_db {
|
||||
DatabaseType::Mysql => "TINYINT(1)".into(),
|
||||
DatabaseType::SqlServer => "BIT".into(),
|
||||
_ => "BOOLEAN".into(),
|
||||
},
|
||||
"date" => "DATE".into(),
|
||||
"time" => "TIME".into(),
|
||||
"datetime" => match target_db {
|
||||
DatabaseType::Postgres => "TIMESTAMP".into(),
|
||||
_ => "DATETIME".into(),
|
||||
},
|
||||
"timestamp" | "timestamptz" | "timestamp with time zone"
|
||||
| "timestamp without time zone" => match target_db {
|
||||
DatabaseType::Mysql => "DATETIME".into(),
|
||||
DatabaseType::SqlServer => "DATETIME2".into(),
|
||||
_ => "TIMESTAMP".into(),
|
||||
},
|
||||
"blob" | "longblob" | "mediumblob" | "tinyblob" | "binary" | "varbinary" | "image" => {
|
||||
match target_db {
|
||||
DatabaseType::Postgres => "BYTEA".into(),
|
||||
DatabaseType::Mysql => "BLOB".into(),
|
||||
DatabaseType::SqlServer => "VARBINARY(MAX)".into(),
|
||||
_ => "BLOB".into(),
|
||||
}
|
||||
}
|
||||
"bytea" => match target_db {
|
||||
DatabaseType::Postgres => "BYTEA".into(),
|
||||
DatabaseType::Mysql => "BLOB".into(),
|
||||
_ => "BLOB".into(),
|
||||
},
|
||||
"json" | "jsonb" => match target_db {
|
||||
DatabaseType::Postgres => "JSONB".into(),
|
||||
DatabaseType::Mysql => "JSON".into(),
|
||||
_ => "TEXT".into(),
|
||||
},
|
||||
"uuid" => match target_db {
|
||||
DatabaseType::Postgres => "UUID".into(),
|
||||
_ => "VARCHAR(36)".into(),
|
||||
},
|
||||
"bit" => match target_db {
|
||||
DatabaseType::Postgres => "BOOLEAN".into(),
|
||||
_ => "BIT".into(),
|
||||
},
|
||||
_ => "TEXT".into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn generate_create_table_ddl(
|
||||
columns: &[db::ColumnInfo],
|
||||
table: &str,
|
||||
schema: &str,
|
||||
target_db: &DatabaseType,
|
||||
source_db: &DatabaseType,
|
||||
) -> String {
|
||||
let full_table = qualified_table(table, schema, target_db);
|
||||
|
||||
let col_lines: Vec<String> = columns
|
||||
.iter()
|
||||
.map(|c| {
|
||||
let mapped_type = map_column_type(&c.data_type, source_db, target_db);
|
||||
let mut line = format!(" {} {}", quote_identifier(&c.name, target_db), mapped_type);
|
||||
if !c.is_nullable {
|
||||
line.push_str(" NOT NULL");
|
||||
}
|
||||
line
|
||||
})
|
||||
.collect();
|
||||
|
||||
let pks: Vec<String> = columns
|
||||
.iter()
|
||||
.filter(|c| c.is_primary_key)
|
||||
.map(|c| quote_identifier(&c.name, target_db))
|
||||
.collect();
|
||||
|
||||
let mut ddl = match target_db {
|
||||
DatabaseType::SqlServer => format!("IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '{table}')\n"),
|
||||
_ => String::new(),
|
||||
};
|
||||
|
||||
let create_prefix = match target_db {
|
||||
DatabaseType::SqlServer => "CREATE TABLE",
|
||||
_ => "CREATE TABLE IF NOT EXISTS",
|
||||
};
|
||||
|
||||
ddl.push_str(&format!("{create_prefix} {full_table} (\n"));
|
||||
ddl.push_str(&col_lines.join(",\n"));
|
||||
|
||||
if !pks.is_empty() {
|
||||
ddl.push_str(&format!(",\n PRIMARY KEY ({})", pks.join(", ")));
|
||||
}
|
||||
|
||||
ddl.push_str("\n)");
|
||||
|
||||
if matches!(target_db, DatabaseType::ClickHouse) {
|
||||
ddl.push_str(" ENGINE = MergeTree() ORDER BY tuple()");
|
||||
}
|
||||
|
||||
ddl
|
||||
}
|
||||
|
||||
pub fn generate_insert(
|
||||
columns: &[String],
|
||||
rows: &[Vec<serde_json::Value>],
|
||||
table: &str,
|
||||
schema: &str,
|
||||
db_type: &DatabaseType,
|
||||
) -> String {
|
||||
if rows.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let full_table = qualified_table(table, schema, db_type);
|
||||
let col_list = columns
|
||||
.iter()
|
||||
.map(|c| quote_identifier(c, db_type))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
|
||||
let value_rows: Vec<String> = rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
let vals: Vec<String> = row.iter().map(|v| escape_value(v, db_type)).collect();
|
||||
format!("({})", vals.join(", "))
|
||||
})
|
||||
.collect();
|
||||
|
||||
format!("INSERT INTO {full_table} ({col_list}) VALUES\n{}", value_rows.join(",\n"))
|
||||
}
|
||||
|
||||
pub fn pagination_sql(
|
||||
columns: &[String],
|
||||
table: &str,
|
||||
schema: &str,
|
||||
db_type: &DatabaseType,
|
||||
offset: u64,
|
||||
limit: usize,
|
||||
) -> String {
|
||||
let full_table = qualified_table(table, schema, db_type);
|
||||
let col_list = columns
|
||||
.iter()
|
||||
.map(|c| quote_identifier(c, db_type))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
|
||||
match db_type {
|
||||
DatabaseType::SqlServer | DatabaseType::Oracle => {
|
||||
format!(
|
||||
"SELECT {col_list} FROM {full_table} ORDER BY (SELECT NULL) OFFSET {offset} ROWS FETCH NEXT {limit} ROWS ONLY"
|
||||
)
|
||||
}
|
||||
_ => {
|
||||
format!("SELECT {col_list} FROM {full_table} LIMIT {limit} OFFSET {offset}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn count_sql(table: &str, schema: &str, db_type: &DatabaseType) -> String {
|
||||
let full_table = qualified_table(table, schema, db_type);
|
||||
format!("SELECT COUNT(*) FROM {full_table}")
|
||||
}
|
||||
|
||||
pub async fn execute_on_pool(
|
||||
state: &AppState,
|
||||
pool_key: &str,
|
||||
sql: &str,
|
||||
) -> Result<db::QueryResult, String> {
|
||||
let connections = state.connections.lock().await;
|
||||
let pool = connections.get(pool_key).ok_or("Connection not found")?;
|
||||
|
||||
match pool {
|
||||
PoolKind::Mysql(p, bare) => {
|
||||
let p = p.clone();
|
||||
let bare = *bare;
|
||||
drop(connections);
|
||||
db::mysql::execute_query(&p, sql, bare).await
|
||||
}
|
||||
PoolKind::Postgres(p) => {
|
||||
let p = p.clone();
|
||||
drop(connections);
|
||||
db::postgres::execute_query(&p, sql).await
|
||||
}
|
||||
PoolKind::Sqlite(p) => {
|
||||
let p = p.clone();
|
||||
drop(connections);
|
||||
db::sqlite::execute_query(&p, sql).await
|
||||
}
|
||||
PoolKind::ClickHouse(client) => {
|
||||
let client = client.clone();
|
||||
let database = pool_key.split(':').nth(1).unwrap_or("default").to_string();
|
||||
drop(connections);
|
||||
db::clickhouse_driver::execute_query(&client, &database, sql).await
|
||||
}
|
||||
PoolKind::SqlServer(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
db::sqlserver::execute_query(&mut client, sql).await
|
||||
}
|
||||
PoolKind::Oracle(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
let client = client.lock().await;
|
||||
db::oracle_driver::execute_query(&*client, sql).await
|
||||
}
|
||||
PoolKind::DuckDb(con) => {
|
||||
let con = con.clone();
|
||||
let sql = sql.to_string();
|
||||
drop(connections);
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let con = con.lock().map_err(|e| e.to_string())?;
|
||||
let start = std::time::Instant::now();
|
||||
let trimmed = sql.trim().to_uppercase();
|
||||
if trimmed.starts_with("SELECT") || trimmed.starts_with("SHOW")
|
||||
|| trimmed.starts_with("DESCRIBE") || trimmed.starts_with("WITH")
|
||||
|| trimmed.starts_with("PRAGMA")
|
||||
{
|
||||
let mut stmt = con.prepare(&sql).map_err(|e| e.to_string())?;
|
||||
let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
|
||||
let stmt_ref = rows.as_ref().ok_or("DuckDB statement unavailable")?;
|
||||
let col_count = stmt_ref.column_count();
|
||||
let columns: Vec<String> = (0..col_count)
|
||||
.map(|i| stmt_ref.column_name(i).map(|s| s.to_string()).unwrap_or_else(|_| "?".to_string()))
|
||||
.collect();
|
||||
let mut result_rows = Vec::new();
|
||||
while let Some(row) = rows.next().map_err(|e| e.to_string())? {
|
||||
let vals: Vec<serde_json::Value> = (0..col_count).map(|i| {
|
||||
row.get::<_, String>(i).map(serde_json::Value::String)
|
||||
.or_else(|_| row.get::<_, i64>(i).map(|v| serde_json::Value::Number(v.into())))
|
||||
.or_else(|_| row.get::<_, f64>(i).map(|v| {
|
||||
serde_json::Number::from_f64(v).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null)
|
||||
}))
|
||||
.or_else(|_| row.get::<_, bool>(i).map(serde_json::Value::Bool))
|
||||
.unwrap_or(serde_json::Value::Null)
|
||||
}).collect();
|
||||
result_rows.push(vals);
|
||||
}
|
||||
Ok(db::QueryResult { columns, rows: result_rows, affected_rows: 0, execution_time_ms: start.elapsed().as_millis(), truncated: false })
|
||||
} else {
|
||||
let affected = con.execute(&sql, []).map_err(|e| e.to_string())?;
|
||||
Ok(db::QueryResult { columns: vec![], rows: vec![], affected_rows: affected as u64, execution_time_ms: start.elapsed().as_millis(), truncated: false })
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
}
|
||||
_ => Err("Unsupported database type for transfer".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_db_type(state: &AppState, connection_id: &str) -> Result<DatabaseType, String> {
|
||||
let configs = state.configs.lock().await;
|
||||
configs
|
||||
.get(connection_id)
|
||||
.map(|c| c.db_type.clone())
|
||||
.ok_or_else(|| format!("Connection config not found: {connection_id}"))
|
||||
}
|
||||
|
||||
pub async fn get_columns_for_transfer(
|
||||
state: &AppState,
|
||||
pool_key: &str,
|
||||
_connection_id: &str,
|
||||
database: &str,
|
||||
schema: &str,
|
||||
table: &str,
|
||||
) -> Result<Vec<db::ColumnInfo>, String> {
|
||||
let connections = state.connections.lock().await;
|
||||
|
||||
if let Some(PoolKind::DuckDb(con)) = connections.get(pool_key) {
|
||||
let con = con.clone();
|
||||
drop(connections);
|
||||
let table = table.to_string();
|
||||
return tokio::task::spawn_blocking(move || {
|
||||
let con = con.lock().map_err(|e| e.to_string())?;
|
||||
let mut stmt = con.prepare(
|
||||
"SELECT column_name, data_type, is_nullable, column_default
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'main' AND table_name = ?
|
||||
ORDER BY ordinal_position"
|
||||
).map_err(|e| e.to_string())?;
|
||||
let rows = stmt.query_map([&table], |row| {
|
||||
Ok(db::ColumnInfo {
|
||||
name: row.get::<_, String>(0)?,
|
||||
data_type: row.get::<_, String>(1)?,
|
||||
is_nullable: row.get::<_, String>(2).unwrap_or_default() == "YES",
|
||||
column_default: row.get::<_, Option<String>>(3)?,
|
||||
is_primary_key: false,
|
||||
extra: None,
|
||||
comment: None,
|
||||
numeric_precision: None,
|
||||
numeric_scale: None,
|
||||
character_maximum_length: None,
|
||||
})
|
||||
}).map_err(|e| e.to_string())?;
|
||||
Ok(rows.filter_map(|r| r.ok()).collect())
|
||||
}).await.map_err(|e| e.to_string())?;
|
||||
}
|
||||
|
||||
if let Some(PoolKind::ClickHouse(client)) = connections.get(pool_key) {
|
||||
let client = client.clone();
|
||||
let database = database.to_string();
|
||||
let table = table.to_string();
|
||||
drop(connections);
|
||||
return db::clickhouse_driver::get_columns(&client, &database, &table).await;
|
||||
}
|
||||
if let Some(PoolKind::SqlServer(client)) = connections.get(pool_key) {
|
||||
let client = client.clone();
|
||||
let schema = schema.to_string();
|
||||
let table = table.to_string();
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
return db::sqlserver::get_columns(&mut client, &schema, &table).await;
|
||||
}
|
||||
if let Some(PoolKind::Oracle(client)) = connections.get(pool_key) {
|
||||
let client = client.clone();
|
||||
let schema = schema.to_string();
|
||||
let table = table.to_string();
|
||||
drop(connections);
|
||||
let client = client.lock().await;
|
||||
return db::oracle_driver::get_columns(&*client, &schema, &table).await;
|
||||
}
|
||||
|
||||
let pool = connections.get(pool_key).ok_or("Pool not found")?;
|
||||
let schema = schema.to_string();
|
||||
let table = table.to_string();
|
||||
match pool {
|
||||
PoolKind::Mysql(p, _) => {
|
||||
let p = p.clone();
|
||||
drop(connections);
|
||||
db::mysql::get_columns(&p, &schema, &table).await
|
||||
}
|
||||
PoolKind::Postgres(p) => {
|
||||
let p = p.clone();
|
||||
drop(connections);
|
||||
db::postgres::get_columns(&p, &schema, &table).await
|
||||
}
|
||||
PoolKind::Sqlite(p) => {
|
||||
let p = p.clone();
|
||||
drop(connections);
|
||||
db::sqlite::get_columns(&p, &schema, &table).await
|
||||
}
|
||||
_ => Err("Unsupported database type".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn is_cancelled(transfer_id: &str) -> bool {
|
||||
CANCELLED.read().await.contains(transfer_id)
|
||||
}
|
||||
|
||||
pub async fn set_cancelled(transfer_id: &str) {
|
||||
CANCELLED.write().await.insert(transfer_id.to_string());
|
||||
}
|
||||
|
||||
pub async fn clear_cancelled(transfer_id: &str) {
|
||||
CANCELLED.write().await.remove(transfer_id);
|
||||
}
|
||||
|
||||
/// Transfer a single table. Returns rows transferred.
|
||||
/// `progress_callback` is invoked for progress updates.
|
||||
pub async fn transfer_table<F>(
|
||||
state: &AppState,
|
||||
request: &TransferRequest,
|
||||
table: &str,
|
||||
table_index: usize,
|
||||
source_db_type: &DatabaseType,
|
||||
target_db_type: &DatabaseType,
|
||||
source_pool_key: &str,
|
||||
target_pool_key: &str,
|
||||
mut progress_callback: F,
|
||||
) -> Result<u64, String>
|
||||
where
|
||||
F: FnMut(TransferProgress),
|
||||
{
|
||||
let total_tables = request.tables.len();
|
||||
|
||||
// Get source columns (deduplicate by name)
|
||||
let columns = {
|
||||
let raw = get_columns_for_transfer(
|
||||
state, source_pool_key, &request.source_connection_id,
|
||||
&request.source_database, &request.source_schema, table,
|
||||
).await?;
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
raw.into_iter().filter(|c| seen.insert(c.name.clone())).collect::<Vec<_>>()
|
||||
};
|
||||
|
||||
if columns.is_empty() {
|
||||
return Err(format!("No columns found for table {table}"));
|
||||
}
|
||||
|
||||
let col_names: Vec<String> = columns.iter().map(|c| c.name.clone()).collect();
|
||||
log::info!("[transfer] {} has {} columns, counting rows...", table, columns.len());
|
||||
|
||||
// Count source rows
|
||||
let total_rows = {
|
||||
let sql = count_sql(table, &request.source_schema, source_db_type);
|
||||
match execute_on_pool(state, source_pool_key, &sql).await {
|
||||
Ok(result) => result.rows.first()
|
||||
.and_then(|r| r.first())
|
||||
.and_then(|v| match v {
|
||||
serde_json::Value::Number(n) => n.as_u64(),
|
||||
serde_json::Value::String(s) => s.parse::<u64>().ok(),
|
||||
_ => None,
|
||||
}),
|
||||
Err(e) => {
|
||||
log::warn!("[transfer] count failed for {}: {}", table, e);
|
||||
None
|
||||
}
|
||||
}
|
||||
};
|
||||
log::info!("[transfer] {} total_rows={:?}", table, total_rows);
|
||||
|
||||
// Create table on target if requested
|
||||
if request.create_table {
|
||||
let ddl = generate_create_table_ddl(&columns, table, &request.target_schema, target_db_type, source_db_type);
|
||||
log::info!("[transfer] creating target table: {}", &ddl[..ddl.len().min(200)]);
|
||||
if let Err(e) = execute_on_pool(state, target_pool_key, &ddl).await {
|
||||
let err_lower = e.to_lowercase();
|
||||
if !err_lower.contains("already exists") && !err_lower.contains("there is already") {
|
||||
return Err(format!("Failed to create table: {e}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Truncate target if requested
|
||||
if request.truncate_before {
|
||||
let full_table = qualified_table(table, &request.target_schema, target_db_type);
|
||||
let truncate_sql = match target_db_type {
|
||||
DatabaseType::Sqlite | DatabaseType::DuckDb => format!("DELETE FROM {full_table}"),
|
||||
_ => format!("TRUNCATE TABLE {full_table}"),
|
||||
};
|
||||
execute_on_pool(state, target_pool_key, &truncate_sql).await
|
||||
.map_err(|e| format!("Failed to truncate: {e}"))?;
|
||||
}
|
||||
|
||||
// Transfer data in batches
|
||||
let batch_size = if request.batch_size == 0 { 1000 } else { request.batch_size };
|
||||
let mut offset: u64 = 0;
|
||||
let mut total_transferred: u64 = 0;
|
||||
|
||||
loop {
|
||||
if is_cancelled(&request.transfer_id).await {
|
||||
return Err("Cancelled".to_string());
|
||||
}
|
||||
|
||||
let sql = pagination_sql(&col_names, table, &request.source_schema, source_db_type, offset, batch_size);
|
||||
let result = execute_on_pool(state, source_pool_key, &sql).await?;
|
||||
let row_count = result.rows.len();
|
||||
|
||||
if row_count == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
let insert_sql = generate_insert(&col_names, &result.rows, table, &request.target_schema, target_db_type);
|
||||
if !insert_sql.is_empty() {
|
||||
execute_on_pool(state, target_pool_key, &insert_sql).await
|
||||
.map_err(|e| format!("Insert failed at offset {offset}: {e}"))?;
|
||||
}
|
||||
|
||||
total_transferred += row_count as u64;
|
||||
log::info!("[transfer] {} batch +{} rows (total {})", table, row_count, total_transferred);
|
||||
offset += row_count as u64;
|
||||
|
||||
progress_callback(TransferProgress {
|
||||
transfer_id: request.transfer_id.clone(),
|
||||
table: table.to_string(),
|
||||
table_index,
|
||||
total_tables,
|
||||
rows_transferred: total_transferred,
|
||||
total_rows,
|
||||
status: TransferStatus::Running,
|
||||
error: None,
|
||||
});
|
||||
|
||||
if row_count < batch_size {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(total_transferred)
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct DatabaseInfo {
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TableInfo {
|
||||
pub name: String,
|
||||
pub table_type: String, // "TABLE" or "VIEW"
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ColumnInfo {
|
||||
pub name: String,
|
||||
pub data_type: String,
|
||||
pub is_nullable: bool,
|
||||
pub column_default: Option<String>,
|
||||
pub is_primary_key: bool,
|
||||
pub extra: Option<String>,
|
||||
pub comment: Option<String>,
|
||||
pub numeric_precision: Option<i32>,
|
||||
pub numeric_scale: Option<i32>,
|
||||
pub character_maximum_length: Option<i32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct QueryResult {
|
||||
pub columns: Vec<String>,
|
||||
pub rows: Vec<Vec<serde_json::Value>>,
|
||||
pub affected_rows: u64,
|
||||
pub execution_time_ms: u128,
|
||||
#[serde(default)]
|
||||
pub truncated: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct IndexInfo {
|
||||
pub name: String,
|
||||
pub columns: Vec<String>,
|
||||
pub is_unique: bool,
|
||||
pub is_primary: bool,
|
||||
pub filter: Option<String>,
|
||||
pub index_type: Option<String>,
|
||||
pub included_columns: Option<Vec<String>>,
|
||||
pub comment: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ForeignKeyInfo {
|
||||
pub name: String,
|
||||
pub column: String,
|
||||
pub ref_table: String,
|
||||
pub ref_column: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TriggerInfo {
|
||||
pub name: String,
|
||||
pub event: String,
|
||||
pub timing: String,
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub const LATEST_RELEASE_URL: &str = "https://api.github.com/repos/t8y2/dbx/releases/latest";
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct GithubRelease {
|
||||
pub tag_name: String,
|
||||
pub name: Option<String>,
|
||||
pub html_url: String,
|
||||
pub body: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct UpdateInfo {
|
||||
pub current_version: String,
|
||||
pub latest_version: String,
|
||||
pub update_available: bool,
|
||||
pub release_name: String,
|
||||
pub release_url: String,
|
||||
pub release_notes: String,
|
||||
}
|
||||
|
||||
pub async fn fetch_latest_release() -> Result<GithubRelease, String> {
|
||||
let client = reqwest::Client::new();
|
||||
client
|
||||
.get(LATEST_RELEASE_URL)
|
||||
.header(reqwest::header::USER_AGENT, "dbx-update-checker")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("Failed to check updates: {e}"))?
|
||||
.error_for_status()
|
||||
.map_err(|e| format!("Failed to check updates: {e}"))?
|
||||
.json::<GithubRelease>()
|
||||
.await
|
||||
.map_err(|e| format!("Failed to parse update response: {e}"))
|
||||
}
|
||||
|
||||
pub fn build_update_info(release: GithubRelease, current_version: &str) -> UpdateInfo {
|
||||
let latest_version = normalize_version(&release.tag_name);
|
||||
|
||||
UpdateInfo {
|
||||
update_available: is_newer_version(&latest_version, current_version),
|
||||
current_version: current_version.to_string(),
|
||||
latest_version,
|
||||
release_name: release.name.unwrap_or_else(|| release.tag_name.clone()),
|
||||
release_url: release.html_url,
|
||||
release_notes: release.body.unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn normalize_version(version: &str) -> String {
|
||||
version.trim().trim_start_matches('v').to_string()
|
||||
}
|
||||
|
||||
pub fn parse_version(version: &str) -> Vec<u64> {
|
||||
normalize_version(version)
|
||||
.split(['.', '-', '+'])
|
||||
.map(|part| part.parse::<u64>().unwrap_or(0))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn is_newer_version(latest: &str, current: &str) -> bool {
|
||||
let latest_parts = parse_version(latest);
|
||||
let current_parts = parse_version(current);
|
||||
let max_len = latest_parts.len().max(current_parts.len());
|
||||
|
||||
for i in 0..max_len {
|
||||
let latest_part = *latest_parts.get(i).unwrap_or(&0);
|
||||
let current_part = *current_parts.get(i).unwrap_or(&0);
|
||||
if latest_part > current_part {
|
||||
return true;
|
||||
}
|
||||
if latest_part < current_part {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{is_newer_version, normalize_version};
|
||||
|
||||
#[test]
|
||||
fn normalizes_tag_versions() {
|
||||
assert_eq!(normalize_version("v1.2.3"), "1.2.3");
|
||||
assert_eq!(normalize_version(" 0.2.0 "), "0.2.0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compares_semver_like_versions() {
|
||||
assert!(is_newer_version("0.2.1", "0.2.0"));
|
||||
assert!(is_newer_version("1.0.0", "0.9.9"));
|
||||
assert!(!is_newer_version("0.2.0", "0.2.0"));
|
||||
assert!(!is_newer_version("0.1.9", "0.2.0"));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user