feat(duckdb): move support to standalone driver

This commit is contained in:
t8y2 committed 2026-07-30 00:50:39 +08:00
1 parent ed018dbc77
commit d347c77c3d
76 files changed
+5400 -8258

No files matched your search

+9
View File
@@ -40,12 +40,14 @@ const H2_PROFILES: &[AgentDriverProfile] =
&[AgentDriverProfile { profile: "h2-legacy", key: "h2-legacy", label: "H2 2.1 Legacy", store_visible: true }];
const EXTRA_AGENT_LABELS: &[(&str, &str)] = &[
("duckdb", "DuckDB"),
("kafka", "Apache Kafka"),
("rocketmq", "Apache RocketMQ"),
("rabbitmq", "RabbitMQ"),
("sqlserver-legacy", "SQL Server legacy compatibility component"),
];
const EXTRA_DRIVER_STORE_ENTRIES: &[(&str, &str)] = &[
("duckdb", "DuckDB"),
("kafka", "Apache Kafka"),
("rocketmq", "Apache RocketMQ"),
("rabbitmq", "RabbitMQ"),
@@ -388,4 +390,11 @@ mod tests {
assert_eq!(agent_key(&DatabaseType::H2, Some("h2-legacy")), Some("h2-legacy"));
assert!(driver_store_entries().any(|(key, label)| key == "h2-legacy" && label == "H2 2.1 Legacy"));
}
#[test]
fn duckdb_is_available_in_driver_store_without_using_agent_runtime() {
assert!(driver_store_entries().any(|(key, label)| key == "duckdb" && label == "DuckDB"));
assert_eq!(label_for_key("duckdb"), Some("DuckDB"));
assert!(!is_agent_type(&DatabaseType::DuckDb));
}
}
+8
View File
@@ -345,6 +345,14 @@ pub struct DriverInfo {
pub struct ArtifactInfo {
pub url: String,
pub size: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub format: Option<ArtifactFormat>,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum ArtifactFormat {
TarZstd,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+670 -30
View File
@@ -8,7 +8,7 @@ use futures::{stream, StreamExt};
use crate::agent_catalog;
use crate::agent_manager::{
AgentDriverInfo, AgentManager, AgentRegistry, InstalledDriver, JavaRuntimeMode, DEFAULT_JRE_KEY,
AgentDriverInfo, AgentManager, AgentRegistry, ArtifactFormat, InstalledDriver, JavaRuntimeMode, DEFAULT_JRE_KEY,
};
use crate::DownloadSource;
@@ -577,7 +577,7 @@ pub async fn reinstall_agent_jre_from(
.platforms
.get(platform)
.ok_or_else(|| format!("No JRE {jre_key} available for platform: {platform}"))?;
let jre_archive = am.base_dir().join("jre-download.tar.gz");
let jre_archive = jre_archive_download_path(am, jre_key, platform_jre.format);
download_with_progress(
am,
&progress,
@@ -600,7 +600,7 @@ pub async fn reinstall_agent_jre_from(
am.stop_daemons().await;
let stash = replace_old_jre_dir(&jre_dir)?;
persist_pending_jre_cleanup(am, stash.as_ref()).await?;
extract_tar_gz(&jre_archive, &jre_dir)?;
extract_jre_archive(&jre_archive, &jre_dir, platform_jre.format)?;
std::fs::remove_file(&jre_archive).ok();
am.mutate_state(|state| state.jre_versions.insert(jre_key.to_string(), jre_info.version.clone()))?;
cleanup_jre_download_cache_after_success(am, jre_key);
@@ -627,6 +627,41 @@ pub async fn import_agents_from_zip(
.await
}
pub fn inspect_offline_package(package_path: &Path) -> Result<OfflineImportPlan, String> {
if is_tar_zstd_package(package_path) {
inspect_tar_zstd_driver_package(package_path)
} else {
inspect_offline_zip(package_path)
}
}
pub async fn import_agents_from_package(
am: &AgentManager,
package_path: &Path,
progress: impl Fn(AgentProgressEvent),
) -> Result<OfflineImportResult, String> {
if is_tar_zstd_package(package_path) {
import_tar_zstd_driver_package(am, package_path, |event| {
progress(AgentProgressEvent {
operation_id: None,
step: event.step,
downloaded: Some(event.current as u64),
total: Some(event.total as u64),
db_type: event.db_type,
current: Some(event.current),
total_drivers: Some(event.total),
});
})
.await
} else {
import_agents_from_zip(am, package_path, progress).await
}
}
fn is_tar_zstd_package(path: &Path) -> bool {
path.file_name().and_then(|name| name.to_str()).is_some_and(|name| name.to_ascii_lowercase().ends_with(".tar.zst"))
}
async fn install_agent_driver_with_batch(
am: &AgentManager,
db_type: &str,
@@ -734,7 +769,7 @@ async fn ensure_jre_from_registry(
.platforms
.get(platform)
.ok_or_else(|| format!("No JRE {jre_key} available for platform: {platform}"))?;
let jre_archive = am.base_dir().join(format!("jre-{}-download.tar.gz", jre_key));
let jre_archive = jre_archive_download_path(am, jre_key, platform_jre.format);
progress(AgentProgressEvent::transfer("jre", 0, platform_jre.size).with_batch(
Some(db_type),
current,
@@ -764,11 +799,11 @@ async fn ensure_jre_from_registry(
let stash = replace_old_jre_dir(&jre_dir)?;
// Persist the stash path *before* extraction so that a crash during
// extract_tar_gz (or a process kill) doesn't leave the renamed-stash
// archive extraction (or a process kill) doesn't leave the renamed-stash
// directory as an orphan that never gets cleaned up.
persist_pending_jre_cleanup(am, stash.as_ref()).await?;
extract_tar_gz(&jre_archive, &jre_dir)?;
extract_jre_archive(&jre_archive, &jre_dir, platform_jre.format)?;
std::fs::remove_file(&jre_archive).ok();
cleanup_jre_download_cache_after_success(am, jre_key);
@@ -893,6 +928,8 @@ async fn install_agent_driver_from_registry(
if let Some(parent) = target_path.parent() {
std::fs::create_dir_all(parent).map_err(|err| format!("Failed to create driver directory: {err}"))?;
}
let artifact_kind = if is_native_artifact { DriverArtifactKind::Native } else { DriverArtifactKind::Jar };
let download_path = driver_artifact_download_path(&target_path, artifact.format);
progress(AgentProgressEvent::transfer("driver", 0, artifact.size).with_batch(
Some(db_type),
current,
@@ -905,7 +942,7 @@ async fn install_agent_driver_from_registry(
source,
&artifact.url,
&r2_path_with_cache_buster(&github_url_to_r2_path(&artifact.url, "driver"), &driver.version),
&target_path,
&download_path,
artifact.size,
Some(CacheIdentity::Driver { db_type, version: &driver.version }),
Some(db_type),
@@ -913,16 +950,24 @@ async fn install_agent_driver_from_registry(
total_drivers,
)
.await?;
install_downloaded_driver_artifact(
&download_path,
&target_path,
artifact.format,
artifact_kind,
db_type,
&driver.version,
)?;
// Some drivers publish both a native agent and a legacy JAR fallback. Only
// validate the artifact type that was actually installed.
if !is_native_artifact && !am.is_driver_jar_valid(db_type) {
std::fs::remove_file(&target_path).ok();
return Err(format!("Downloaded driver jar is invalid or corrupt: {}", target_path.display()));
}
if is_native_artifact {
mark_executable(&target_path)?;
std::fs::remove_file(am.driver_jar_path(db_type)).ok();
} else {
if !am.is_driver_jar_valid(db_type) {
std::fs::remove_file(&target_path).ok();
return Err(format!("Downloaded driver jar is invalid or corrupt: {}", target_path.display()));
}
std::fs::remove_file(am.driver_native_path(db_type)).ok();
}
@@ -947,6 +992,169 @@ async fn install_agent_driver_from_registry(
Ok(())
}
fn driver_artifact_download_path(target_path: &Path, format: Option<ArtifactFormat>) -> PathBuf {
match format {
Some(ArtifactFormat::TarZstd) => {
let file_name = target_path.file_name().and_then(|name| name.to_str()).unwrap_or("agent");
target_path.with_file_name(format!(".{file_name}.tar.zst"))
}
None => target_path.to_path_buf(),
}
}
fn install_downloaded_driver_artifact(
download_path: &Path,
target_path: &Path,
format: Option<ArtifactFormat>,
artifact_kind: DriverArtifactKind,
db_type: &str,
expected_version: &str,
) -> Result<(), String> {
let Some(format) = format else {
return Ok(());
};
let result = match format {
ArtifactFormat::TarZstd => {
install_driver_from_tar_zstd_package(download_path, target_path, artifact_kind, db_type, expected_version)
}
};
if result.is_ok() {
std::fs::remove_file(download_path).ok();
}
result
}
fn install_driver_from_tar_zstd_package(
package_path: &Path,
target_path: &Path,
expected_kind: DriverArtifactKind,
db_type: &str,
expected_version: &str,
) -> Result<(), String> {
let info = tar_zstd_driver_package_info(package_path)?;
if info.db_type != db_type {
return Err(format!("Driver package contains {}, expected {db_type}", info.db_type));
}
if info.version != expected_version {
return Err(format!(
"Driver package version mismatch for {db_type}: expected {expected_version}, got {}",
info.version
));
}
if info.kind != expected_kind {
return Err(format!(
"Driver package artifact type mismatch for {db_type}: expected {}, got {}",
expected_kind.label(),
info.kind.label()
));
}
let parent = target_path.parent().ok_or_else(|| format!("Invalid driver path: {}", target_path.display()))?;
std::fs::create_dir_all(parent).map_err(|error| format!("Failed to create driver directory: {error}"))?;
let staging_path = parent.join(format!(".agent-package-{}", uuid::Uuid::new_v4()));
let result = extract_tar_zstd_file(package_path, &info.entry_name, &staging_path, info.size).and_then(|_| {
match info.kind {
DriverArtifactKind::Jar if !is_valid_agent_jar(&staging_path) => {
return Err(format!("Packaged driver jar is invalid or corrupt: {}", info.entry_name));
}
DriverArtifactKind::Native => {
validate_native_agent_binary(&staging_path)?;
mark_executable(&staging_path)?;
}
DriverArtifactKind::Jar => {}
}
replace_download(&staging_path, target_path)
});
if result.is_err() {
std::fs::remove_file(&staging_path).ok();
}
result
}
fn read_registry_from_tar_zstd(package_path: &Path) -> Result<AgentRegistry, String> {
const MAX_REGISTRY_BYTES: u64 = 1024 * 1024;
let file = std::fs::File::open(package_path).map_err(|error| format!("Failed to open driver package: {error}"))?;
let decoder = zstd::stream::read::Decoder::new(file)
.map_err(|error| format!("Failed to open zstd driver package: {error}"))?;
let mut archive = tar::Archive::new(decoder);
let entries = archive.entries().map_err(|error| format!("Invalid tar.zst driver package: {error}"))?;
for entry in entries {
let mut entry = entry.map_err(|error| format!("Invalid tar.zst driver package entry: {error}"))?;
let path = entry.path().map_err(|error| format!("Invalid driver package path: {error}"))?;
let name = safe_archive_entry_name(&path)?;
if name != "agent-registry.json" {
continue;
}
if !entry.header().entry_type().is_file() {
return Err("Driver package registry is not a regular file".to_string());
}
if entry.size() > MAX_REGISTRY_BYTES {
return Err("Driver package registry is too large".to_string());
}
let mut json = String::new();
entry.read_to_string(&mut json).map_err(|error| format!("Failed to read driver package registry: {error}"))?;
return serde_json::from_str(&json).map_err(|error| format!("Invalid driver package registry: {error}"));
}
Err("agent-registry.json not found in the driver package".to_string())
}
fn extract_tar_zstd_file(
package_path: &Path,
expected_entry: &str,
destination: &Path,
expected_size: u64,
) -> Result<(), String> {
let file = std::fs::File::open(package_path).map_err(|error| format!("Failed to open driver package: {error}"))?;
let decoder = zstd::stream::read::Decoder::new(file)
.map_err(|error| format!("Failed to open zstd driver package: {error}"))?;
let mut archive = tar::Archive::new(decoder);
let entries = archive.entries().map_err(|error| format!("Invalid tar.zst driver package: {error}"))?;
let mut extracted = false;
for entry in entries {
let mut entry = entry.map_err(|error| format!("Invalid tar.zst driver package entry: {error}"))?;
let path = entry.path().map_err(|error| format!("Invalid driver package path: {error}"))?;
let name = safe_archive_entry_name(&path)?;
if name == "agent-registry.json" || entry.header().entry_type().is_dir() {
continue;
}
if name != expected_entry {
return Err(format!("Unexpected file in driver package: {name}"));
}
if extracted || !entry.header().entry_type().is_file() {
return Err(format!("Invalid driver package entry: {name}"));
}
let mut output =
std::fs::File::create(destination).map_err(|error| format!("Failed to create staged driver: {error}"))?;
let copied = std::io::copy(&mut entry, &mut output)
.map_err(|error| format!("Failed to extract driver package: {error}"))?;
std::io::Write::flush(&mut output).map_err(|error| format!("Failed to flush staged driver: {error}"))?;
if expected_size > 0 && copied != expected_size {
return Err(format!("Packaged driver size mismatch: expected {expected_size} bytes, got {copied} bytes"));
}
extracted = true;
}
if extracted {
Ok(())
} else {
Err(format!("Driver package entry not found: {expected_entry}"))
}
}
fn safe_archive_entry_name(path: &Path) -> Result<String, String> {
let mut parts = Vec::new();
for component in path.components() {
match component {
std::path::Component::Normal(value) => parts.push(value.to_string_lossy().into_owned()),
std::path::Component::CurDir => {}
_ => return Err(format!("Unsafe driver package path: {}", path.display())),
}
}
if parts.is_empty() {
return Err("Driver package contains an empty path".to_string());
}
Ok(parts.join("/"))
}
fn agent_registry_driver<'a>(
registry: &'a AgentRegistry,
db_type: &str,
@@ -1416,10 +1624,181 @@ pub struct OfflineImportPlan {
pub includes_jre: bool,
}
type OfflineJreEntry = (String, String);
type OfflineJreEntry = (String, String, Option<ArtifactFormat>);
type OfflineDriverEntry = (String, String, bool);
type OfflineArchiveEntries = (Vec<OfflineJreEntry>, Vec<OfflineDriverEntry>);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum DriverArtifactKind {
Jar,
Native,
}
impl DriverArtifactKind {
fn label(self) -> &'static str {
match self {
Self::Jar => "jar",
Self::Native => "native",
}
}
}
#[derive(Debug, Clone)]
struct TarZstdDriverPackageInfo {
db_type: String,
version: String,
jre: String,
kind: DriverArtifactKind,
entry_name: String,
size: u64,
}
fn inspect_tar_zstd_driver_package(package_path: &Path) -> Result<OfflineImportPlan, String> {
let info = tar_zstd_driver_package_info(package_path)?;
Ok(OfflineImportPlan { driver_keys: vec![info.db_type], includes_jre: false })
}
async fn import_tar_zstd_driver_package(
am: &AgentManager,
package_path: &Path,
progress: impl Fn(OfflineImportProgress),
) -> Result<OfflineImportResult, String> {
let _installation_guard = am.installation_operation_lock.write().await;
let info = tar_zstd_driver_package_info(package_path)?;
let mut result =
OfflineImportResult { jre_installed: Vec::new(), drivers_installed: Vec::new(), drivers_skipped: Vec::new() };
if let Some(installed) = am.load_state().installed_drivers.get(&info.db_type) {
if installed.version != "0.1.0-local"
&& installed.version != "local"
&& !crate::update::is_newer_version(&info.version, &installed.version)
{
result.drivers_skipped.push(info.db_type);
return Ok(result);
}
}
progress(OfflineImportProgress {
step: "driver".to_string(),
current: 1,
total: 1,
label: agent_catalog::label_for_key(&info.db_type).unwrap_or(&info.db_type).to_string(),
db_type: Some(info.db_type.clone()),
});
let target_path = match info.kind {
DriverArtifactKind::Jar => am.driver_jar_path(&info.db_type),
DriverArtifactKind::Native => am.driver_native_path(&info.db_type),
};
install_driver_from_tar_zstd_package(package_path, &target_path, info.kind, &info.db_type, &info.version)?;
match info.kind {
DriverArtifactKind::Jar => {
std::fs::remove_file(am.driver_native_path(&info.db_type)).ok();
}
DriverArtifactKind::Native => {
std::fs::remove_file(am.driver_jar_path(&info.db_type)).ok();
}
}
am.mutate_state(|state| {
state.installed_drivers.insert(
info.db_type.clone(),
InstalledDriver {
version: info.version.clone(),
installed_at: chrono::Utc::now().to_rfc3339(),
jre: info.jre.clone(),
},
);
})?;
am.stop_daemon_by_key(&info.db_type).await;
result.drivers_installed.push(info.db_type);
Ok(result)
}
fn tar_zstd_driver_package_info(package_path: &Path) -> Result<TarZstdDriverPackageInfo, String> {
let registry = read_registry_from_tar_zstd(package_path)?;
if registry.drivers.len() != 1 {
return Err("A tar.zst driver package must contain exactly one driver".to_string());
}
let (db_type, driver) = registry.drivers.iter().next().expect("checked one driver");
validate_offline_driver_key(db_type)?;
let platform = AgentManager::current_platform();
let native_artifact = driver.native.get(platform);
let jar_artifact = driver.jar.as_ref();
let (kind, artifact) = match (native_artifact, jar_artifact) {
(Some(_), Some(_)) => {
return Err("A tar.zst driver package must contain exactly one driver artifact".to_string());
}
(Some(artifact), None) => (DriverArtifactKind::Native, artifact),
(None, Some(artifact)) => (DriverArtifactKind::Jar, artifact),
(None, None) if !driver.native.is_empty() => {
return Err(format!("Driver package does not support platform: {platform}"));
}
(None, None) => return Err("A tar.zst driver package contains no driver artifact".to_string()),
};
if artifact.format.is_some() {
return Err("Nested driver packages are not supported".to_string());
}
let artifact_filename = artifact
.url
.rsplit('/')
.next()
.filter(|name| !name.is_empty())
.ok_or_else(|| format!("Invalid packaged driver URL: {}", artifact.url))?;
let entry_name = format!("drivers/{artifact_filename}");
validate_tar_zstd_package_entries(package_path, &entry_name, artifact.size)?;
Ok(TarZstdDriverPackageInfo {
db_type: db_type.clone(),
version: driver.version.clone(),
jre: driver.jre.clone(),
kind,
entry_name,
size: artifact.size,
})
}
fn validate_tar_zstd_package_entries(
package_path: &Path,
expected_entry: &str,
expected_size: u64,
) -> Result<(), String> {
let file = std::fs::File::open(package_path).map_err(|error| format!("Failed to open driver package: {error}"))?;
let decoder = zstd::stream::read::Decoder::new(file)
.map_err(|error| format!("Failed to open zstd driver package: {error}"))?;
let mut archive = tar::Archive::new(decoder);
let entries = archive.entries().map_err(|error| format!("Invalid tar.zst driver package: {error}"))?;
let mut registry_seen = false;
let mut driver_seen = false;
for entry in entries {
let entry = entry.map_err(|error| format!("Invalid tar.zst driver package entry: {error}"))?;
let path = entry.path().map_err(|error| format!("Invalid driver package path: {error}"))?;
let name = safe_archive_entry_name(&path)?;
if entry.header().entry_type().is_dir() {
continue;
}
if !entry.header().entry_type().is_file() {
return Err(format!("Driver package contains a non-regular entry: {name}"));
}
match name.as_str() {
"agent-registry.json" if !registry_seen => registry_seen = true,
value if value == expected_entry && !driver_seen => {
if expected_size > 0 && entry.size() != expected_size {
return Err(format!(
"Packaged driver size mismatch: expected {expected_size} bytes, got {} bytes",
entry.size()
));
}
driver_seen = true;
}
_ => return Err(format!("Unexpected file in driver package: {name}")),
}
}
if !registry_seen {
return Err("agent-registry.json not found in the driver package".to_string());
}
if !driver_seen {
return Err(format!("Driver package entry not found: {expected_entry}"));
}
Ok(())
}
pub fn inspect_offline_zip(zip_path: &Path) -> Result<OfflineImportPlan, String> {
let file = std::fs::File::open(zip_path).map_err(|e| format!("Failed to open ZIP file: {e}"))?;
let mut archive = zip::ZipArchive::new(file).map_err(|e| format!("Invalid ZIP file: {e}"))?;
@@ -1461,7 +1840,7 @@ pub async fn import_offline_zip(
validate_offline_driver_entries(am, &mut archive, &driver_entries)?;
let mut current: u32 = 0;
for (jre_key, entry_name) in &jre_entries {
for (jre_key, entry_name, format) in &jre_entries {
current += 1;
let jre_version = registry.resolve_jre(jre_key).map(|j| j.version.clone());
let existing_version = local_state.jre_versions.get(jre_key);
@@ -1478,7 +1857,7 @@ pub async fn import_offline_zip(
});
let mut entry = archive.by_name(entry_name).map_err(|e| format!("Failed to read {entry_name}: {e}"))?;
let tmp_archive = am.base_dir().join(format!("jre-offline-{jre_key}.tar.gz"));
let tmp_archive = am.base_dir().join(format!("jre-offline-{jre_key}{}", jre_archive_suffix(*format)));
{
let mut out =
std::fs::File::create(&tmp_archive).map_err(|e| format!("Failed to create temp file: {e}"))?;
@@ -1487,7 +1866,7 @@ pub async fn import_offline_zip(
let jre_dir = am.jre_dir(jre_key);
let staging_dir = am.base_dir().join(format!(".jre-offline-import-{}", uuid::Uuid::new_v4()));
if let Err(error) = extract_tar_gz(&tmp_archive, &staging_dir) {
if let Err(error) = extract_jre_archive(&tmp_archive, &staging_dir, *format) {
std::fs::remove_dir_all(&staging_dir).ok();
std::fs::remove_file(&tmp_archive).ok();
return Err(error);
@@ -1599,7 +1978,7 @@ fn collect_offline_entries(
registry: &AgentRegistry,
) -> Result<OfflineArchiveEntries, String> {
let platform = AgentManager::current_platform();
let mut jre_entries = Vec::new();
let mut jres = std::collections::BTreeMap::<String, (String, Option<ArtifactFormat>)>::new();
let mut drivers = std::collections::BTreeMap::<String, (String, bool)>::new();
for index in 0..archive.len() {
@@ -1608,11 +1987,21 @@ fn collect_offline_entries(
return Err(format!("Offline package contains an unsafe path: {}", entry.name()));
};
let name = path.to_string_lossy().replace('\\', "/");
if name.starts_with("jre/") && name.ends_with(".tar.gz") && name.contains(platform) {
if name.starts_with("jre/") && name.contains(platform) {
let jre_format = if name.ends_with(".tar.zst") {
Some(ArtifactFormat::TarZstd)
} else if name.ends_with(".tar.gz") {
None
} else {
continue;
};
let jre_key = extract_jre_key_from_filename(&name)
.ok_or_else(|| format!("Invalid JRE filename in offline package: {name}"))?;
validate_offline_identifier(&jre_key, "JRE")?;
jre_entries.push((jre_key, name));
let replace = !jres.contains_key(&jre_key) || jre_format == Some(ArtifactFormat::TarZstd);
if replace {
jres.insert(jre_key, (name, jre_format));
}
} else if name.starts_with("drivers/") && name.ends_with(".jar") {
let db_type = db_type_for_jar_offline_entry(registry, &name)
.or_else(|| extract_db_type_from_filename(&name))
@@ -1629,7 +2018,10 @@ fn collect_offline_entries(
}
}
Ok((jre_entries, drivers.into_iter().map(|(db_type, (name, is_native))| (db_type, name, is_native)).collect()))
Ok((
jres.into_iter().map(|(jre_key, (name, format))| (jre_key, name, format)).collect(),
drivers.into_iter().map(|(db_type, (name, is_native))| (db_type, name, is_native)).collect(),
))
}
fn validate_offline_driver_entries(
@@ -1723,7 +2115,36 @@ fn db_type_for_jar_offline_entry(registry: &AgentRegistry, name: &str) -> Option
})
}
fn jre_archive_suffix(format: Option<ArtifactFormat>) -> &'static str {
match format {
Some(ArtifactFormat::TarZstd) => ".tar.zst",
None => ".tar.gz",
}
}
fn jre_archive_download_path(am: &AgentManager, jre_key: &str, format: Option<ArtifactFormat>) -> PathBuf {
am.base_dir().join(format!("jre-{jre_key}-download{}", jre_archive_suffix(format)))
}
fn extract_jre_archive(archive: &Path, dest: &Path, format: Option<ArtifactFormat>) -> Result<(), String> {
match format {
Some(ArtifactFormat::TarZstd) => {
let file = std::fs::File::open(archive).map_err(|e| format!("Failed to open JRE archive: {e}"))?;
let decoder =
zstd::stream::read::Decoder::new(file).map_err(|e| format!("Failed to open zstd JRE archive: {e}"))?;
extract_jre_tar(tar::Archive::new(decoder), dest)
}
None => extract_tar_gz(archive, dest),
}
}
fn extract_tar_gz(archive: &Path, dest: &Path) -> Result<(), String> {
let file = std::fs::File::open(archive).map_err(|e| format!("Failed to open JRE archive: {e}"))?;
let decoder = flate2::read::GzDecoder::new(file);
extract_jre_tar(tar::Archive::new(decoder), dest)
}
fn extract_jre_tar<R: Read>(mut archive: tar::Archive<R>, dest: &Path) -> Result<(), String> {
let parent = dest.parent().ok_or_else(|| format!("Invalid JRE destination: {}", dest.display()))?;
std::fs::create_dir_all(parent).map_err(|e| format!("Failed to create JRE directory: {e}"))?;
@@ -1731,9 +2152,7 @@ fn extract_tar_gz(archive: &Path, dest: &Path) -> Result<(), String> {
.prefix(".jre-extract-")
.tempdir_in(parent)
.map_err(|e| format!("Failed to create JRE extraction directory: {e}"))?;
let file = std::fs::File::open(archive).map_err(|e| format!("Failed to open JRE archive: {e}"))?;
let decoder = flate2::read::GzDecoder::new(file);
tar::Archive::new(decoder).unpack(staging.path()).map_err(|e| format!("Failed to extract JRE archive: {e}"))?;
archive.unpack(staging.path()).map_err(|e| format!("Failed to extract JRE archive: {e}"))?;
let mut roots = std::fs::read_dir(staging.path())
.map_err(|e| format!("Failed to inspect extracted JRE archive: {e}"))?
@@ -2090,7 +2509,7 @@ mod agent_download_url_tests {
#[cfg(test)]
mod agent_registry_install_tests {
use super::*;
use crate::agent_manager::{ArtifactInfo, DriverInfo, InstalledDriver, JavaRuntimeConfig, JreInfo};
use crate::agent_manager::{ArtifactFormat, ArtifactInfo, DriverInfo, InstalledDriver, JavaRuntimeConfig, JreInfo};
fn test_manager(name: &str) -> AgentManager {
let dir = std::env::temp_dir().join(format!("dbx-agent-registry-install-{name}-{}", uuid::Uuid::new_v4()));
@@ -2124,10 +2543,11 @@ mod agent_registry_install_tests {
jar: Some(ArtifactInfo {
url: format!("https://example.com/dbx-agent-{db_type}-legacy-placeholder.jar"),
size: 0,
format: None,
}),
native: [(
AgentManager::current_platform().to_string(),
ArtifactInfo { url: native_url.to_string(), size: native_size },
ArtifactInfo { url: native_url.to_string(), size: native_size, format: None },
)]
.into_iter()
.collect(),
@@ -2145,7 +2565,7 @@ mod agent_registry_install_tests {
label: db_type.to_string(),
min_app_version: "0.1.0".to_string(),
jre: DEFAULT_JRE_KEY.to_string(),
jar: Some(ArtifactInfo { url: url.to_string(), size }),
jar: Some(ArtifactInfo { url: url.to_string(), size, format: None }),
native: std::collections::HashMap::new(),
},
);
@@ -2180,6 +2600,110 @@ mod agent_registry_install_tests {
cache_path
}
fn current_platform_native_binary() -> Vec<u8> {
if cfg!(windows) {
let mut bytes = vec![0_u8; 0x48];
bytes[..2].copy_from_slice(b"MZ");
bytes[0x3c..0x40].copy_from_slice(&(0x40_u32).to_le_bytes());
bytes[0x40..0x44].copy_from_slice(b"PE\0\0");
let machine = if cfg!(target_arch = "aarch64") { 0xaa64_u16 } else { 0x8664_u16 };
bytes[0x44..0x46].copy_from_slice(&machine.to_le_bytes());
bytes
} else if cfg!(target_os = "linux") {
let mut bytes = vec![0_u8; 20];
bytes[..4].copy_from_slice(b"\x7fELF");
bytes[4] = 2;
bytes[5] = 1;
let machine = if cfg!(target_arch = "aarch64") { 183_u16 } else { 62_u16 };
bytes[18..20].copy_from_slice(&machine.to_le_bytes());
bytes
} else if cfg!(target_os = "macos") {
let mut bytes = vec![0xcf, 0xfa, 0xed, 0xfe];
let cpu_type = if cfg!(target_arch = "aarch64") { 0x0100_000c_u32 } else { 0x0100_0007_u32 };
bytes.extend_from_slice(&cpu_type.to_le_bytes());
bytes
} else {
Vec::new()
}
}
fn test_agent_jar() -> Vec<u8> {
let mut bytes = std::io::Cursor::new(Vec::new());
{
use std::io::Write;
let mut archive = zip::ZipWriter::new(&mut bytes);
archive.start_file("META-INF/MANIFEST.MF", zip::write::SimpleFileOptions::default()).unwrap();
archive.write_all(b"Manifest-Version: 1.0\nMain-Class: com.dbx.Agent\n").unwrap();
archive.finish().unwrap();
}
bytes.into_inner()
}
fn build_tar_zstd_driver_package(
db_type: &str,
version: &str,
kind: DriverArtifactKind,
driver_bytes: &[u8],
) -> Vec<u8> {
let platform = AgentManager::current_platform();
let (filename, artifact) = match kind {
DriverArtifactKind::Jar => (
format!("dbx-agent-{db_type}-{version}.jar"),
serde_json::json!({
"jar": {
"url": format!("dbx-agent-{db_type}-{version}.jar"),
"size": driver_bytes.len()
}
}),
),
DriverArtifactKind::Native => {
let extension = if platform.starts_with("windows-") { ".exe" } else { "" };
let filename = format!("dbx-agent-{db_type}-{version}-{platform}{extension}");
(
filename.clone(),
serde_json::json!({
"native": {
platform: {
"url": filename,
"size": driver_bytes.len()
}
}
}),
)
}
};
let mut driver = serde_json::json!({
"version": version,
"label": db_type,
"min_app_version": "0.6.0",
"jre": DEFAULT_JRE_KEY
});
driver.as_object_mut().unwrap().extend(artifact.as_object().unwrap().clone());
let registry = serde_json::json!({
"jres": {},
"drivers": {
db_type: driver
}
});
let registry_bytes = registry.to_string().into_bytes();
let encoder = zstd::stream::write::Encoder::new(Vec::new(), 3).unwrap();
let mut archive = tar::Builder::new(encoder);
let mut registry_header = tar::Header::new_gnu();
registry_header.set_size(registry_bytes.len() as u64);
registry_header.set_mode(0o644);
registry_header.set_cksum();
archive.append_data(&mut registry_header, "agent-registry.json", registry_bytes.as_slice()).unwrap();
let mut driver_header = tar::Header::new_gnu();
driver_header.set_size(driver_bytes.len() as u64);
driver_header.set_mode(if kind == DriverArtifactKind::Native { 0o755 } else { 0o644 });
driver_header.set_cksum();
archive.append_data(&mut driver_header, format!("drivers/{filename}"), driver_bytes).unwrap();
archive.into_inner().unwrap().finish().unwrap()
}
#[test]
fn managed_jre_content_revision_triggers_install() {
let manager = test_manager("jre-content-revision");
@@ -2208,7 +2732,7 @@ mod agent_registry_install_tests {
version: version.to_string(),
platforms: [(
AgentManager::current_platform().to_string(),
ArtifactInfo { url: url.to_string(), size },
ArtifactInfo { url: url.to_string(), size, format: None },
)]
.into_iter()
.collect(),
@@ -2234,8 +2758,29 @@ mod agent_registry_install_tests {
builder.into_inner().unwrap().finish().unwrap()
}
fn write_cached_jre_download(am: &AgentManager, jre_key: &str, version: &str, url: &str, archive: &[u8]) {
let dest = am.base_dir().join(format!("jre-{jre_key}-download.tar.gz"));
fn build_zstd_jre_archive(am: &AgentManager, jre_key: &str) -> Vec<u8> {
let archive_root = am.base_dir().join("jre-zstd-test-archive");
let payload = archive_root.join("payload");
let java_path = am.jre_java_path(jre_key);
let relative_java_path = java_path.strip_prefix(am.jre_dir(jre_key)).unwrap();
let java_path = payload.join(relative_java_path);
std::fs::create_dir_all(java_path.parent().unwrap()).unwrap();
std::fs::write(java_path, b"java").unwrap();
let encoder = zstd::stream::write::Encoder::new(Vec::new(), 3).unwrap();
let mut builder = tar::Builder::new(encoder);
builder.append_dir_all("payload", &payload).unwrap();
builder.into_inner().unwrap().finish().unwrap()
}
fn write_cached_jre_download(
am: &AgentManager,
jre_key: &str,
version: &str,
url: &str,
format: Option<ArtifactFormat>,
archive: &[u8],
) {
let dest = jre_archive_download_path(am, jre_key, format);
let cache_path = cached_download_path(
am,
url,
@@ -2284,6 +2829,81 @@ mod agent_registry_install_tests {
.any(|event| event.step == "done" && event.db_type.as_deref() == Some(db_type)));
}
#[tokio::test]
async fn registry_install_extracts_tar_zstd_native_driver_package() {
let manager = test_manager("tar-zstd-native-package");
let db_type = "duckdb";
let version = "0.1.0";
let package_url = "https://example.com/dbx-agent-duckdb.tar.zst";
let native_bytes = current_platform_native_binary();
let package_bytes = build_tar_zstd_driver_package(db_type, version, DriverArtifactKind::Native, &native_bytes);
let mut registry =
registry_with_native_and_legacy_jar(db_type, version, package_url, package_bytes.len() as u64);
registry.drivers.get_mut(db_type).unwrap().native.get_mut(AgentManager::current_platform()).unwrap().format =
Some(ArtifactFormat::TarZstd);
let native_path = manager.driver_native_path(db_type);
let package_path = driver_artifact_download_path(&native_path, Some(ArtifactFormat::TarZstd));
let cache_path =
write_cached_driver_download(&manager, db_type, version, package_url, &package_path, &package_bytes);
let progress = |_| {};
install_agent_driver_from_registry(
&manager,
&registry,
DownloadSource::Official,
db_type,
&progress,
None,
None,
)
.await
.unwrap();
assert_eq!(std::fs::read(&native_path).unwrap(), native_bytes);
assert!(!package_path.exists());
assert!(!cache_path.exists());
assert_eq!(manager.load_state().installed_drivers[db_type].version, version);
}
#[tokio::test]
async fn registry_install_extracts_tar_zstd_java_driver_package() {
let manager = test_manager("tar-zstd-java-package");
let db_type = "dameng";
let version = "0.2.0";
let package_url = "https://example.com/dbx-agent-dameng.tar.zst";
let jar_bytes = test_agent_jar();
let package_bytes = build_tar_zstd_driver_package(db_type, version, DriverArtifactKind::Jar, &jar_bytes);
let mut registry = registry_with_jar(db_type, version, package_url, package_bytes.len() as u64);
registry.drivers.get_mut(db_type).unwrap().jar.as_mut().unwrap().format = Some(ArtifactFormat::TarZstd);
manager
.mutate_state(|state| {
state.java_runtime = JavaRuntimeConfig { mode: JavaRuntimeMode::System, custom_java_path: None };
})
.unwrap();
let jar_path = manager.driver_jar_path(db_type);
let package_path = driver_artifact_download_path(&jar_path, Some(ArtifactFormat::TarZstd));
let cache_path =
write_cached_driver_download(&manager, db_type, version, package_url, &package_path, &package_bytes);
let progress = |_| {};
install_agent_driver_from_registry(
&manager,
&registry,
DownloadSource::Official,
db_type,
&progress,
None,
None,
)
.await
.unwrap();
assert_eq!(std::fs::read(&jar_path).unwrap(), jar_bytes);
assert!(!package_path.exists());
assert!(!cache_path.exists());
assert_eq!(manager.load_state().installed_drivers[db_type].version, version);
}
#[tokio::test]
async fn batch_upgrade_preserves_successful_state_and_reports_independent_failure() {
let manager = test_manager("batch-upgrade");
@@ -2367,7 +2987,7 @@ mod agent_registry_install_tests {
let events = std::sync::Mutex::new(Vec::new());
let progress = |event| events.lock().unwrap().push(event);
write_cached_jre_download(&manager, jre_key, version, url, &archive);
write_cached_jre_download(&manager, jre_key, version, url, None, &archive);
ensure_jre_from_registry(
&manager,
&registry,
@@ -2383,7 +3003,7 @@ mod agent_registry_install_tests {
// The successful install cleans its cache. Re-add it so the old
// implementation fails deterministically by consuming it again.
write_cached_jre_download(&manager, jre_key, version, url, &archive);
write_cached_jre_download(&manager, jre_key, version, url, None, &archive);
ensure_jre_from_registry(
&manager,
&registry,
@@ -2410,6 +3030,26 @@ mod agent_registry_install_tests {
.any(|event| event.step == "jre" && event.db_type.as_deref() == Some("dameng")));
}
#[tokio::test]
async fn managed_jre_install_extracts_tar_zstd_archive() {
let manager = test_manager("managed-zstd-jre");
let jre_key = DEFAULT_JRE_KEY;
let version = "21.0.12";
let url = "https://example.com/dbx-jre.tar.zst";
let archive = build_zstd_jre_archive(&manager, jre_key);
let mut registry = registry_with_jre(jre_key, version, url, archive.len() as u64);
registry.jres.get_mut(jre_key).unwrap().platforms.get_mut(AgentManager::current_platform()).unwrap().format =
Some(ArtifactFormat::TarZstd);
write_cached_jre_download(&manager, jre_key, version, url, Some(ArtifactFormat::TarZstd), &archive);
ensure_jre_from_registry(&manager, &registry, DownloadSource::Official, jre_key, "dameng", &|_| {}, None, None)
.await
.unwrap();
assert!(manager.is_jre_installed(jre_key));
assert_eq!(manager.load_state().jre_versions[jre_key], version);
}
#[tokio::test]
async fn stash_is_recorded_before_jre_extraction() {
let manager = test_manager("stash-before-extract");
+55 -326
View File
@@ -4,8 +4,6 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::{watch, Mutex, RwLock};
#[cfg(feature = "duckdb-bundled")]
use tokio::task::JoinHandle;
use mysql_async::prelude::Queryable;
use mysql_async::Row as MysqlRow;
@@ -42,21 +40,16 @@ const ACCESS_AGENT_CONNECT_TIMEOUT_SECS: u64 = 30;
const POOL_CLOSE_TIMEOUT_SECS: u64 = 3;
const HEALTH_CHECK_POOL_ACQUIRE_TIMEOUT: Duration = Duration::from_millis(500);
#[cfg(feature = "duckdb-bundled")]
mod duckdb_types {
#[cfg(feature = "duckdb-sidecar")]
use std::sync::Arc;
pub type DuckDbHandle = Arc<crate::db::duckdb_driver::DuckDbConnection>;
#[cfg(feature = "duckdb-sidecar")]
pub type DuckDbWorkerHandle = Arc<crate::db::duckdb_worker_process::DuckDbWorkerClient>;
pub type ExternalTabularHandle = Arc<crate::external::ExternalPool>;
}
#[cfg(not(feature = "duckdb-bundled"))]
mod duckdb_types {
pub type DuckDbHandle = ();
#[cfg(not(feature = "duckdb-sidecar"))]
pub type DuckDbWorkerHandle = ();
pub type ExternalTabularHandle = ();
}
use duckdb_types::{DuckDbHandle, DuckDbWorkerHandle, ExternalTabularHandle};
use duckdb_types::DuckDbWorkerHandle;
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum MysqlMode {
@@ -100,7 +93,6 @@ pub enum PoolKind {
Turso(db::turso_driver::TursoClient),
CloudflareD1(db::cloudflare_d1_driver::CloudflareD1Client),
Redis(db::redis_driver::RedisConnection),
DuckDb(DuckDbHandle),
DuckDbWorker(DuckDbWorkerHandle),
MongoDb(mongodb::Client),
ClickHouse(db::clickhouse_driver::ChClient),
@@ -110,7 +102,6 @@ pub enum PoolKind {
VectorDb(db::vector_driver::VectorClient),
InfluxDb(db::influxdb_driver::InfluxdbClient),
Agent(Arc<tokio::sync::Mutex<db::agent_driver::AgentDriverClient>>),
ExternalTabular(ExternalTabularHandle),
ExternalDriver {
driver_id: String,
config: Arc<ConnectionConfig>,
@@ -690,41 +681,44 @@ impl AppState {
}
}
#[cfg(feature = "duckdb-bundled")]
async fn create_duckdb_pool(&self, config: &ConnectionConfig) -> Result<PoolKind, String> {
if self.duckdb_worker_process_isolation.load(Ordering::Relaxed) {
let attached_databases = config
.attached_databases
.iter()
.map(|attached| crate::models::connection::AttachedDatabaseConfig {
name: attached.name.clone(),
path: expand_tilde(&attached.path),
})
.collect();
let client = db::duckdb_worker_process::DuckDbWorkerClient::open_with_process_limit(
expand_tilde(&config.host),
#[cfg(feature = "duckdb-sidecar")]
pub async fn create_duckdb_pool(&self, config: &ConnectionConfig) -> Result<PoolKind, String> {
let attached_databases = config
.attached_databases
.iter()
.map(|attached| crate::models::connection::AttachedDatabaseConfig {
name: attached.name.clone(),
path: expand_tilde(&attached.path),
})
.collect();
let path = expand_tilde(&config.host);
let init_script = config.init_script.clone();
let process_limit = self.duckdb_worker_max_processes.load(Ordering::Relaxed);
let installed_driver = self.agent_manager.driver_native_path("duckdb");
let has_driver_override =
std::env::var_os(db::duckdb_worker_process::DUCKDB_DRIVER_PATH_ENV).is_some_and(|value| !value.is_empty());
let client = if installed_driver.is_file() && !has_driver_override {
db::duckdb_worker_process::DuckDbWorkerClient::open_with_executable_and_process_limit(
installed_driver,
path,
attached_databases,
config.init_script.clone(),
self.duckdb_worker_max_processes.load(Ordering::Relaxed),
init_script,
process_limit,
)
.await?;
Ok(PoolKind::DuckDbWorker(Arc::new(client)))
.await?
} else {
let con = db::duckdb_driver::connect_path(&expand_tilde(&config.host))?;
{
let locked = con.lock().map_err(|e| e.to_string())?;
for attached in &config.attached_databases {
crate::schema::duckdb_attach_database(&locked, &attached.name, &expand_tilde(&attached.path))?;
}
if let Some(script) = config.init_script.as_deref() {
db::duckdb_driver::run_init_script(&locked, script)?;
}
}
Ok(PoolKind::DuckDb(con))
}
db::duckdb_worker_process::DuckDbWorkerClient::open_with_process_limit(
path,
attached_databases,
init_script,
process_limit,
)
.await?
};
Ok(PoolKind::DuckDbWorker(Arc::new(client)))
}
#[cfg(feature = "duckdb-bundled")]
#[cfg(feature = "duckdb-sidecar")]
pub async fn test_duckdb_connection_config(&self, config: &ConnectionConfig) -> Result<(), String> {
// Test the submitted form as a fresh session so unsaved ATTACH/init
// changes cannot be masked by a pool created from older settings.
@@ -1315,11 +1309,11 @@ impl AppState {
};
PoolKind::Redis(con)
}
#[cfg(feature = "duckdb-bundled")]
#[cfg(feature = "duckdb-sidecar")]
DatabaseType::DuckDb => self.create_duckdb_pool(&db_config).await?,
#[cfg(not(feature = "duckdb-bundled"))]
#[cfg(not(feature = "duckdb-sidecar"))]
DatabaseType::DuckDb => {
return Err("DuckDB support is not compiled in this build. Rebuild with default features.".to_string());
return Err("DuckDB support is not compiled in this build.".to_string());
}
DatabaseType::MongoDb => {
if mongo_uses_legacy_driver(&db_config) {
@@ -2342,9 +2336,7 @@ impl AppState {
}
}
PoolKind::Sqlite(_)
| PoolKind::DuckDb(_)
| PoolKind::DuckDbWorker(_)
| PoolKind::ExternalTabular(_)
| PoolKind::ExternalDriver { .. }
| PoolKind::MessageQueue
| PoolKind::Nacos => false,
@@ -2567,123 +2559,10 @@ impl AppState {
}
}
#[cfg(feature = "duckdb-bundled")]
async fn remove_pool_by_key_detached(&self, pool_key: &str) -> bool {
self.stop_keepalive_task(pool_key).await;
self.pool_activity.write().await.remove(pool_key);
self.postgres_cancel_contexts.write().await.remove(pool_key);
let removed = self.connections.write().await.remove(pool_key);
if let Some(pool) = removed {
close_removed_pools_in_background(&self.task_supervisor, vec![(pool_key.to_string(), pool)]);
true
} else {
false
}
}
#[cfg(feature = "duckdb-bundled")]
async fn remove_pool_if_duckdb_isolation_mismatch(&self, pool_key: &str) -> bool {
let isolation_enabled = self.duckdb_worker_process_isolation.load(Ordering::Relaxed);
let mismatch = {
let connections = self.connections.read().await;
match connections.get(pool_key) {
Some(PoolKind::DuckDb(_)) => isolation_enabled,
Some(PoolKind::DuckDbWorker(_)) => !isolation_enabled,
_ => false,
}
};
if mismatch {
self.remove_pool_by_key_detached(pool_key).await
} else {
false
}
}
#[cfg(not(feature = "duckdb-bundled"))]
async fn remove_pool_if_duckdb_isolation_mismatch(&self, _pool_key: &str) -> bool {
false
}
#[cfg(feature = "duckdb-bundled")]
pub fn spawn_duckdb_pool_cleanup(&self, pool_key: String, con: DuckDbHandle) {
let connections = self.connections.clone();
let supervisor = self.task_supervisor.clone();
let pool_activity = self.pool_activity.clone();
let postgres_cancel_contexts = self.postgres_cancel_contexts.clone();
let task_key = format!("duckdb-cleanup:{pool_key}");
supervisor.clone().spawn_replace(task_key, move |shutdown| async move {
while Arc::strong_count(&con) > 2 {
tokio::select! {
_ = shutdown.cancelled() => return,
_ = tokio::time::sleep(Duration::from_millis(50)) => {}
}
}
supervisor.stop(&format!("keepalive:{pool_key}"));
pool_activity.write().await.remove(&pool_key);
postgres_cancel_contexts.write().await.remove(&pool_key);
let removed = {
let mut conns = connections.write().await;
match conns.get(&pool_key) {
Some(PoolKind::DuckDb(current)) if Arc::ptr_eq(current, &con) => conns.remove(&pool_key),
_ => None,
}
};
if let Some(pool) = removed {
// Keep the old DuckDB pool marked as draining until it is no longer
// visible in the pool map, otherwise a concurrent query could reuse it.
con.clear_draining();
drop(con);
close_pool_kind_with_timeout(pool_key, pool).await;
}
});
}
#[cfg(feature = "duckdb-bundled")]
pub fn spawn_duckdb_draining_cleanup(
&self,
pool_key: String,
con: DuckDbHandle,
mut task: JoinHandle<Result<db::QueryResult, String>>,
) {
let connections = self.connections.clone();
let supervisor = self.task_supervisor.clone();
let pool_activity = self.pool_activity.clone();
let postgres_cancel_contexts = self.postgres_cancel_contexts.clone();
let task_key = format!("duckdb-draining:{pool_key}");
supervisor.clone().spawn_replace(task_key, move |shutdown| async move {
tokio::select! {
_ = shutdown.cancelled() => {
task.abort();
return;
}
_ = &mut task => {}
}
while Arc::strong_count(&con) > 2 {
tokio::select! {
_ = shutdown.cancelled() => return,
_ = tokio::time::sleep(Duration::from_millis(50)) => {}
}
}
supervisor.stop(&format!("keepalive:{pool_key}"));
pool_activity.write().await.remove(&pool_key);
postgres_cancel_contexts.write().await.remove(&pool_key);
let removed = {
let mut conns = connections.write().await;
match conns.get(&pool_key) {
Some(PoolKind::DuckDb(current)) if Arc::ptr_eq(current, &con) => conns.remove(&pool_key),
_ => None,
}
};
if let Some(pool) = removed {
// Keep the old DuckDB pool marked as draining until it is no longer
// visible in the pool map, otherwise a concurrent query could reuse it.
con.clear_draining();
drop(con);
close_pool_kind_with_timeout(pool_key, pool).await;
}
});
}
pub async fn close_database_pool(&self, connection_id: &str, database: Option<&str>) -> Result<bool, String> {
let db_type = {
let configs = self.configs.read().await;
@@ -2732,6 +2611,11 @@ impl AppState {
let mut keys = HashSet::new();
for (pool_key, pool) in connections.iter() {
#[cfg(feature = "duckdb-sidecar")]
if matches!(pool, PoolKind::DuckDbWorker(_)) {
keys.insert("duckdb".to_string());
continue;
}
if !matches!(pool, PoolKind::Agent(_)) {
continue;
}
@@ -3092,9 +2976,7 @@ impl AppState {
}
}
PoolKind::Sqlite(_)
| PoolKind::DuckDb(_)
| PoolKind::DuckDbWorker(_)
| PoolKind::ExternalTabular(_)
| PoolKind::ExternalDriver { .. }
| PoolKind::MessageQueue
| PoolKind::Nacos => true,
@@ -3162,13 +3044,13 @@ impl AppState {
close_removed_pools_in_background(&self.task_supervisor, removed);
}
#[cfg(feature = "duckdb-bundled")]
#[cfg(feature = "duckdb-sidecar")]
async fn remove_duckdb_pools_detached(&self) {
let removed = self.drain_duckdb_pools().await;
close_removed_pools_in_background(&self.task_supervisor, removed);
}
#[cfg(not(feature = "duckdb-bundled"))]
#[cfg(not(feature = "duckdb-sidecar"))]
async fn remove_duckdb_pools_detached(&self) {}
pub async fn remove_external_driver_pools(&self, driver_id: &str) {
@@ -3209,16 +3091,16 @@ impl AppState {
removed
}
#[cfg(feature = "duckdb-bundled")]
#[cfg(feature = "duckdb-sidecar")]
async fn drain_duckdb_pools(&self) -> Vec<(String, PoolKind)> {
let keys_to_remove: Vec<String> = self
.connections
.read()
.await
.iter()
.filter_map(|(key, pool)| match pool {
PoolKind::DuckDb(_) | PoolKind::DuckDbWorker(_) => Some(key.clone()),
_ => None,
.filter_map(|(key, pool)| {
let is_duckdb = matches!(pool, PoolKind::DuckDbWorker(_));
is_duckdb.then(|| key.clone())
})
.collect();
self.stop_keepalive_tasks(&keys_to_remove).await;
@@ -3516,13 +3398,9 @@ fn clone_pool_kind(pool: &PoolKind) -> PoolKind {
PoolKind::Rqlite(client) => PoolKind::Rqlite(client.clone()),
PoolKind::Turso(client) => PoolKind::Turso(client.clone()),
PoolKind::CloudflareD1(client) => PoolKind::CloudflareD1(client.clone()),
#[cfg(feature = "duckdb-bundled")]
PoolKind::DuckDb(con) => PoolKind::DuckDb(con.clone()),
#[cfg(feature = "duckdb-bundled")]
#[cfg(feature = "duckdb-sidecar")]
PoolKind::DuckDbWorker(client) => PoolKind::DuckDbWorker(client.clone()),
#[cfg(not(feature = "duckdb-bundled"))]
PoolKind::DuckDb(_) => PoolKind::DuckDb(()),
#[cfg(not(feature = "duckdb-bundled"))]
#[cfg(not(feature = "duckdb-sidecar"))]
PoolKind::DuckDbWorker(_) => PoolKind::DuckDbWorker(()),
PoolKind::MongoDb(client) => PoolKind::MongoDb(client.clone()),
PoolKind::ClickHouse(client) => PoolKind::ClickHouse(client.clone()),
@@ -3532,10 +3410,6 @@ fn clone_pool_kind(pool: &PoolKind) -> PoolKind {
PoolKind::VectorDb(client) => PoolKind::VectorDb(client.clone()),
PoolKind::InfluxDb(client) => PoolKind::InfluxDb(client.clone()),
PoolKind::Agent(client) => PoolKind::Agent(client.clone()),
#[cfg(feature = "duckdb-bundled")]
PoolKind::ExternalTabular(ext) => PoolKind::ExternalTabular(ext.clone()),
#[cfg(not(feature = "duckdb-bundled"))]
PoolKind::ExternalTabular(_) => PoolKind::ExternalTabular(()),
PoolKind::ExternalDriver { driver_id, config, session } => {
PoolKind::ExternalDriver { driver_id: driver_id.clone(), config: config.clone(), session: session.clone() }
}
@@ -3558,17 +3432,11 @@ pub async fn close_pool_kind(pool: PoolKind) {
PoolKind::Redis(conn) => {
drop(conn);
}
#[cfg(feature = "duckdb-bundled")]
PoolKind::DuckDb(con) => {
crate::db::duckdb_driver::close_connection(con);
}
#[cfg(feature = "duckdb-bundled")]
#[cfg(feature = "duckdb-sidecar")]
PoolKind::DuckDbWorker(client) => {
client.shutdown().await;
}
#[cfg(not(feature = "duckdb-bundled"))]
PoolKind::DuckDb(_) => {}
#[cfg(not(feature = "duckdb-bundled"))]
#[cfg(not(feature = "duckdb-sidecar"))]
PoolKind::DuckDbWorker(_) => {}
PoolKind::MongoDb(client) => {
drop(client);
@@ -3595,7 +3463,6 @@ pub async fn close_pool_kind(pool: PoolKind) {
let mut client = client.lock().await;
let _ = client.disconnect().await;
}
PoolKind::ExternalTabular(_) => {}
PoolKind::ExternalDriver { session, .. } => {
session.shutdown().await;
}
@@ -5108,51 +4975,6 @@ mod tests {
let _ = std::fs::remove_dir_all(dir);
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test]
async fn duckdb_connection_test_does_not_reuse_stale_pool_config() {
let (state, dir) = test_app_state().await;
let db_path = dir.join("app.duckdb");
duckdb::Connection::open(&db_path).unwrap();
let mut config = mysql_config(None);
config.id = "duckdb-conn".to_string();
config.name = "DuckDB".to_string();
config.db_type = DatabaseType::DuckDb;
config.host = db_path.to_string_lossy().to_string();
config.port = 0;
state.configs.write().await.insert(config.id.clone(), config.clone());
state.get_or_create_pool("duckdb-conn", None).await.unwrap();
config.init_script = Some("SELECT definitely_invalid_syntax(".to_string());
let error = state.test_duckdb_connection_config(&config).await.expect_err("invalid submitted script must fail");
assert!(error.contains("Connection init script statement 1 failed"));
let _ = std::fs::remove_dir_all(dir);
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test]
async fn duckdb_connection_test_validates_attached_databases() {
let (state, dir) = test_app_state().await;
let mut config = mysql_config(None);
config.id = "duckdb-conn".to_string();
config.name = "DuckDB".to_string();
config.db_type = DatabaseType::DuckDb;
config.host = ":memory:".to_string();
config.port = 0;
config.attached_databases.push(AttachedDatabaseConfig {
name: "missing".to_string(),
path: dir.join("missing-parent").join("missing.duckdb").to_string_lossy().to_string(),
});
let error = state.test_duckdb_connection_config(&config).await.expect_err("invalid attach must fail");
assert!(error.to_ascii_lowercase().contains("attach"));
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn sqlite_connection_restores_attached_databases() {
let (state, dir) = test_app_state().await;
@@ -5189,72 +5011,6 @@ mod tests {
let _ = std::fs::remove_dir_all(dir);
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test]
async fn duckdb_connection_test_supports_memory_bootstrap() {
let (state, dir) = test_app_state().await;
let mut config = mysql_config(None);
config.id = "duckdb-memory".to_string();
config.name = "DuckDB memory".to_string();
config.db_type = DatabaseType::DuckDb;
config.host = ":memory:".to_string();
config.port = 0;
config.init_script = Some(r#"CREATE TABLE probe AS SELECT E'it\'s;ok' AS value;"#.to_string());
state.test_duckdb_connection_config(&config).await.expect("memory bootstrap succeeds");
let _ = std::fs::remove_dir_all(dir);
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test]
async fn applying_duckdb_worker_isolation_drops_existing_duckdb_pools() {
let (state, dir) = test_app_state().await;
let db_path = dir.join("app.duckdb");
duckdb::Connection::open(&db_path).unwrap();
let mut config = mysql_config(None);
config.id = "duckdb-conn".to_string();
config.name = "DuckDB".to_string();
config.db_type = DatabaseType::DuckDb;
config.host = db_path.to_string_lossy().to_string();
config.port = 0;
state.configs.write().await.insert(config.id.clone(), config);
state.get_or_create_pool("duckdb-conn", None).await.unwrap();
assert!(matches!(state.connections.read().await.get("duckdb-conn"), Some(PoolKind::DuckDb(_))));
state.apply_duckdb_worker_process_isolation(true).await;
assert!(!state.connections.read().await.contains_key("duckdb-conn"));
let _ = std::fs::remove_dir_all(dir);
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test]
async fn duckdb_pool_mode_mismatch_removes_existing_pool() {
let (state, dir) = test_app_state().await;
let db_path = dir.join("app.duckdb");
duckdb::Connection::open(&db_path).unwrap();
let mut config = mysql_config(None);
config.id = "duckdb-conn".to_string();
config.name = "DuckDB".to_string();
config.db_type = DatabaseType::DuckDb;
config.host = db_path.to_string_lossy().to_string();
config.port = 0;
state.configs.write().await.insert(config.id.clone(), config);
state.get_or_create_pool("duckdb-conn", None).await.unwrap();
assert!(matches!(state.connections.read().await.get("duckdb-conn"), Some(PoolKind::DuckDb(_))));
state.set_duckdb_worker_process_isolation_enabled(true);
assert!(state.remove_pool_if_duckdb_isolation_mismatch("duckdb-conn").await);
assert!(!state.connections.read().await.contains_key("duckdb-conn"));
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn remove_connection_pools_clears_base_and_database_scoped_pools() {
let (state, dir) = test_app_state().await;
@@ -5519,33 +5275,6 @@ for line in sys.stdin:
assert!(!super::is_agent_validate_connection_unsupported("Agent RPC error (-1): Unknown method: kv_put"));
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test]
async fn duckdb_client_session_reuses_base_pool_to_avoid_file_locks() {
let (state, dir) = test_app_state().await;
let db_path = dir.join("session.duckdb");
duckdb::Connection::open(&db_path).unwrap();
let mut config = mysql_config(None);
config.id = "duckdb-conn".to_string();
config.name = "DuckDB".to_string();
config.db_type = DatabaseType::DuckDb;
config.host = db_path.to_string_lossy().to_string();
config.port = 0;
state.configs.write().await.insert(config.id.clone(), config.clone());
let base_pool_key = state.get_or_create_pool("duckdb-conn", None).await.unwrap();
let pool_key = state.get_or_create_pool_for_session("duckdb-conn", Some("main"), Some("tab-1")).await.unwrap();
assert_eq!(pool_key, base_pool_key);
assert!(!state.close_client_session_pool("duckdb-conn", Some("main"), "tab-1").await.unwrap());
let conns = state.connections.read().await;
assert!(conns.contains_key("duckdb-conn"));
assert!(!conns.contains_key("duckdb-conn:session:tab-1"));
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn sqlite_memory_client_sessions_share_the_same_database() {
let (state, dir) = test_app_state().await;
-198
View File
@@ -1,198 +0,0 @@
#![cfg(feature = "duckdb-bundled")]
use std::fs;
use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::sync::Mutex;
pub struct DuckDbConnection {
connection: Mutex<duckdb::Connection>,
interrupt_handle: Arc<duckdb::InterruptHandle>,
draining: AtomicBool,
}
impl DuckDbConnection {
pub fn new(connection: duckdb::Connection) -> Self {
let interrupt_handle = connection.interrupt_handle();
Self { connection: Mutex::new(connection), interrupt_handle, draining: AtomicBool::new(false) }
}
pub fn lock(&self) -> std::sync::LockResult<std::sync::MutexGuard<'_, duckdb::Connection>> {
self.connection.lock()
}
pub fn interrupt_handle(&self) -> Arc<duckdb::InterruptHandle> {
self.interrupt_handle.clone()
}
pub fn mark_draining(&self) {
self.draining.store(true, Ordering::SeqCst);
}
pub fn clear_draining(&self) {
self.draining.store(false, Ordering::SeqCst);
}
pub fn is_draining(&self) -> bool {
self.draining.load(Ordering::SeqCst)
}
// Preserve PoisonError ownership so close_connection can still release the contained DuckDB handle.
#[allow(clippy::result_large_err)]
fn into_inner(self) -> std::sync::LockResult<duckdb::Connection> {
self.connection.into_inner()
}
}
/// Connects to a DuckDb database file with file validation.
///
/// # Arguments
/// * `path` - The file path to the DuckDb database
///
/// # Returns
/// * `Ok(Arc<DuckDbConnection>)` on successful connection
/// * `Err(String)` with descriptive error message if connection fails
pub fn connect_path(path: &str) -> Result<Arc<DuckDbConnection>, String> {
let is_memory = is_memory_database_path(path);
if !is_memory {
validate_duckdb_path(path)?;
remove_empty_placeholder_file(path)?;
}
let connection = if is_memory { duckdb::Connection::open_in_memory() } else { duckdb::Connection::open(path) }
.map_err(|e| format!("DuckDb connection failed: {e}"))?;
Ok(Arc::new(DuckDbConnection::new(connection)))
}
/// Runs a connection init script statement-by-statement so a failure can
/// point at the offending statement instead of the whole batch.
pub fn run_init_script(con: &duckdb::Connection, script: &str) -> Result<(), String> {
for (index, statement) in crate::db::duckdb_sql::split_sql_statements(script).iter().enumerate() {
if crate::db::duckdb_sql::strip_leading_comments(statement).is_empty() {
continue;
}
con.execute_batch(statement)
.map_err(|e| format!("Connection init script statement {} failed: {e}", index + 1))?;
}
Ok(())
}
fn validate_duckdb_path(path: &str) -> Result<(), String> {
if path.is_empty() {
return Err("Database file path cannot be empty".to_string());
}
if path.contains('\0') {
return Err("Database file path contains null characters".to_string());
}
if is_network_path(path) {
return Ok(());
}
let path_obj = Path::new(path);
if path_obj.is_dir() {
return Err(format!("Database file path is a directory, not a file: {}", path));
}
if !path_obj.exists() {
// DuckDB creates a new database file on open when the parent directory
// exists. Only reject paths whose parent directory is missing.
if let Some(parent) = path_obj.parent() {
if !parent.as_os_str().is_empty() && !parent.exists() {
return Err(format!("Parent directory does not exist: {}", parent.display()));
}
}
}
Ok(())
}
fn remove_empty_placeholder_file(path: &str) -> Result<(), String> {
let path_obj = Path::new(path);
if is_network_path(path) || !path_obj.exists() {
return Ok(());
}
let metadata = fs::metadata(path_obj).map_err(|e| format!("Failed to read database file metadata: {e}"))?;
if metadata.is_file() && metadata.len() == 0 {
fs::remove_file(path_obj).map_err(|e| format!("Failed to replace empty DuckDB placeholder file: {e}"))?;
}
Ok(())
}
fn is_network_path(path: &str) -> bool {
path.starts_with("\\\\") || path.starts_with("//") || path.contains("wsl.localhost") || path.contains("wsl$")
}
pub fn is_memory_database_path(path: &str) -> bool {
path.trim().eq_ignore_ascii_case(":memory:")
}
/// Closes a DuckDB connection, releasing the file lock.
///
/// Unlike relying on Drop, this calls `duckdb_disconnect` synchronously
/// so the file handle is released before this function returns.
/// On Windows this prevents "file already in use" errors when reconnecting.
pub fn close_connection(con: Arc<DuckDbConnection>) {
match Arc::try_unwrap(con) {
Ok(handle) => match handle.into_inner() {
Ok(conn) => {
let _ = conn.close();
}
Err(poisoned) => {
let _ = poisoned.into_inner().close();
}
},
Err(_) => {
// Arc still referenced elsewhere (e.g. running queries);
// the last holder will drop and close the connection.
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn connect_path_supports_memory_database() {
let con = connect_path(":memory:").expect("connect in-memory DuckDB");
let locked = con.lock().expect("lock connection");
locked.execute_batch("CREATE TABLE memory_probe AS SELECT 42 AS id;").expect("create table");
let value: i32 = locked.query_row("SELECT id FROM memory_probe;", [], |row| row.get(0)).expect("select row");
assert_eq!(value, 42);
}
#[test]
fn interrupt_handle_is_available_while_connection_is_locked() {
let con = connect_path(":memory:").expect("connect in-memory DuckDB");
let _locked = con.lock().expect("lock connection");
let _interrupt_handle = con.interrupt_handle();
}
#[test]
fn connect_path_replaces_empty_placeholder_file() {
let path = std::env::temp_dir().join(format!("dbx-duckdb-empty-{}.duckdb", uuid::Uuid::new_v4()));
std::fs::write(&path, "").expect("write empty placeholder");
assert_eq!(std::fs::metadata(&path).expect("placeholder metadata").len(), 0);
let con = connect_path(path.to_string_lossy().as_ref()).expect("connect empty placeholder DuckDB file");
close_connection(con);
assert!(std::fs::metadata(&path).expect("database metadata").len() > 0);
let _ = std::fs::remove_file(path);
}
#[test]
fn run_init_script_executes_duckdb_escape_strings() {
let con = connect_path(":memory:").expect("connect in-memory DuckDB");
let locked = con.lock().expect("lock connection");
run_init_script(&locked, r#"CREATE TABLE probe AS SELECT E'it\'s;ok' AS value; SELECT 2;"#)
.expect("run init script");
let value: String = locked.query_row("SELECT value FROM probe", [], |row| row.get(0)).expect("select value");
assert_eq!(value, "it's;ok");
}
}
+3 -3
View File
@@ -1,7 +1,7 @@
//! Quote-aware helpers for inspecting DuckDB SQL snippets without a full
//! parser. Shared by the DuckDB worker (execute-time ATTACH tracking) and the
//! connection layer (init-script handling), and kept free of `duckdb-bundled`
//! so config-derived metadata works in every build.
//! connection layer (init-script handling). It stays independent of the native
//! DuckDB engine so config-derived metadata remains in the main application.
use crate::models::connection::ConnectionConfig;
@@ -130,7 +130,7 @@ pub fn attached_name_from_attach_sql(sql: &str) -> Option<String> {
parse_identifier_after_as(&trimmed[as_index + 2..])
}
pub(crate) fn strip_leading_comments(sql: &str) -> &str {
pub fn strip_leading_comments(sql: &str) -> &str {
let mut rest = sql.trim_start();
loop {
if let Some(after) = rest.strip_prefix("--") {
+89 -13
View File
@@ -1,7 +1,8 @@
#![cfg(feature = "duckdb-bundled")]
#![cfg(feature = "duckdb-sidecar")]
use std::collections::HashMap;
use std::path::PathBuf;
use std::ffi::OsString;
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex as StdMutex, OnceLock};
@@ -14,20 +15,21 @@ use tokio_util::sync::CancellationToken;
use crate::db;
use crate::db::duckdb_worker_protocol::{
DuckDbWorkerConnectParams, DuckDbWorkerError, DuckDbWorkerExecuteParams, DuckDbWorkerMethod,
DuckDbWorkerObjectSourceParams, DuckDbWorkerRequest, DuckDbWorkerResponse,
DuckDbWorkerColumnParams, DuckDbWorkerConnectParams, DuckDbWorkerError, DuckDbWorkerExecuteParams,
DuckDbWorkerMethod, DuckDbWorkerObjectSourceParams, DuckDbWorkerRequest, DuckDbWorkerResponse,
};
use crate::models::connection::AttachedDatabaseConfig;
use crate::storage::{normalize_duckdb_worker_max_processes, DUCKDB_WORKER_MAX_PROCESSES_DEFAULT};
/// Error code the worker reports when a query error left the DuckDB connection poisoned
/// (see the duckdb-rs Parser Error bug). The client kills the worker on this code so the
/// next request starts a fresh one. Must match the code emitted in `duckdb_worker_runtime`.
/// next request starts a fresh one. Must match the code emitted by the standalone driver runtime.
const DUCKDB_WORKER_POISONED_CODE: &str = "duckdb_worker_poisoned";
const DUCKDB_WORKER_REQUEST_TIMEOUT_CODE: &str = "duckdb_worker_request_timeout";
const DEFAULT_WORKER_REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
const DEFAULT_WORKER_KILL_WAIT: Duration = Duration::from_secs(3);
const DEFAULT_WORKER_START_WAIT: Duration = Duration::from_secs(5);
pub const DUCKDB_DRIVER_PATH_ENV: &str = "DBX_DUCKDB_DRIVER_PATH";
type PendingRequests = Arc<Mutex<HashMap<String, PendingRequest>>>;
struct PendingRequest {
@@ -48,6 +50,7 @@ struct DuckDbWorkerClientInner {
process_limiter: Arc<Semaphore>,
process_limit: usize,
executable: PathBuf,
executable_args: Vec<OsString>,
connect_params: DuckDbWorkerConnectParams,
request_timeout: Duration,
worker_start_timeout: Duration,
@@ -68,8 +71,16 @@ impl DuckDbWorkerClient {
attached_databases: Vec<AttachedDatabaseConfig>,
init_script: Option<String>,
) -> Result<Self, String> {
let executable = std::env::current_exe().map_err(|e| e.to_string())?;
Self::open_with_executable(executable, path, attached_databases, init_script).await
let (executable, executable_args) = resolve_duckdb_driver_command()?;
Self::open_with_command_and_process_limit(
executable,
executable_args,
path,
attached_databases,
init_script,
DUCKDB_WORKER_MAX_PROCESSES_DEFAULT,
)
.await
}
pub async fn open_with_process_limit(
@@ -78,9 +89,16 @@ impl DuckDbWorkerClient {
init_script: Option<String>,
process_limit: usize,
) -> Result<Self, String> {
let executable = std::env::current_exe().map_err(|e| e.to_string())?;
Self::open_with_executable_and_process_limit(executable, path, attached_databases, init_script, process_limit)
.await
let (executable, executable_args) = resolve_duckdb_driver_command()?;
Self::open_with_command_and_process_limit(
executable,
executable_args,
path,
attached_databases,
init_script,
process_limit,
)
.await
}
pub async fn open_with_executable(
@@ -105,9 +123,29 @@ impl DuckDbWorkerClient {
attached_databases: Vec<AttachedDatabaseConfig>,
init_script: Option<String>,
process_limit: usize,
) -> Result<Self, String> {
Self::open_with_command_and_process_limit(
executable,
Vec::new(),
path,
attached_databases,
init_script,
process_limit,
)
.await
}
async fn open_with_command_and_process_limit(
executable: PathBuf,
executable_args: Vec<OsString>,
path: String,
attached_databases: Vec<AttachedDatabaseConfig>,
init_script: Option<String>,
process_limit: usize,
) -> Result<Self, String> {
let client = Self::new_unconnected_with_timeouts(
executable,
executable_args,
path,
attached_databases,
init_script,
@@ -122,6 +160,7 @@ impl DuckDbWorkerClient {
#[doc(hidden)]
pub fn new_unconnected_with_timeouts(
executable: PathBuf,
executable_args: Vec<OsString>,
path: String,
attached_databases: Vec<AttachedDatabaseConfig>,
init_script: Option<String>,
@@ -139,6 +178,7 @@ impl DuckDbWorkerClient {
process_limiter,
process_limit,
executable,
executable_args,
connect_params: DuckDbWorkerConnectParams { path, attached_databases, init_script },
request_timeout,
worker_start_timeout,
@@ -246,6 +286,18 @@ impl DuckDbWorkerClient {
.await
}
pub async fn get_table_ddl(&self, database: String, schema: String, table: String) -> Result<String, String> {
self.metadata_request(DuckDbWorkerMethod::GetTableDdl, DuckDbWorkerColumnParams { database, schema, table })
.await
}
pub async fn completion_assistant(
&self,
request: db::CompletionAssistantRequest,
) -> Result<db::CompletionAssistantResponse, String> {
self.metadata_request(DuckDbWorkerMethod::CompletionAssistant, request).await
}
pub async fn get_object_source(
&self,
database: String,
@@ -413,9 +465,14 @@ impl DuckDbWorkerClient {
.map_err(|_| duckdb_worker_process_limit_error(self.inner.process_limit))?
.map_err(|_| "DuckDB worker process limiter is closed".to_string())?;
log::info!("[duckdb-worker:start] executable={}", self.inner.executable.display());
let mut child = crate::process::new_tokio_command(&self.inner.executable)
.arg("--duckdb-worker")
log::info!(
"[duckdb-worker:start] executable={} args={:?}",
self.inner.executable.display(),
self.inner.executable_args
);
let mut command = crate::process::new_tokio_command(&self.inner.executable);
command.args(&self.inner.executable_args);
let mut child = command
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::inherit())
@@ -524,6 +581,25 @@ impl DuckDbWorkerClient {
}
}
fn resolve_duckdb_driver_command() -> Result<(PathBuf, Vec<OsString>), String> {
if let Some(executable) = std::env::var_os(DUCKDB_DRIVER_PATH_ENV).filter(|value| !value.is_empty()) {
let executable = PathBuf::from(executable);
ensure_duckdb_driver_exists(&executable)?;
return Ok((executable, Vec::new()));
}
Err(format!(
"DuckDB driver is not installed. Please install it from the Driver Manager or set {DUCKDB_DRIVER_PATH_ENV}."
))
}
fn ensure_duckdb_driver_exists(executable: &Path) -> Result<(), String> {
if executable.is_file() {
return Ok(());
}
Err(format!("DuckDB driver configured by {DUCKDB_DRIVER_PATH_ENV} was not found: {}", executable.display()))
}
struct WorkerProcessLimiterState {
limit: usize,
limiter: Arc<Semaphore>,
@@ -1,4 +1,4 @@
#![cfg(feature = "duckdb-bundled")]
#![cfg(feature = "duckdb-sidecar")]
use serde::{de::DeserializeOwned, Deserialize, Serialize};
@@ -35,6 +35,8 @@ pub enum DuckDbWorkerMethod {
ListSchemas,
ListTables,
ListColumns,
GetTableDdl,
CompletionAssistant,
GetObjectSource,
AttachDatabase,
Cancel,
@@ -1,642 +0,0 @@
#![cfg(feature = "duckdb-bundled")]
use std::sync::{Arc, Mutex};
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Stdout};
use crate::db;
use crate::db::duckdb_worker_protocol::{
DuckDbWorkerColumnParams, DuckDbWorkerConnectParams, DuckDbWorkerDatabaseParams, DuckDbWorkerError,
DuckDbWorkerExecuteParams, DuckDbWorkerMethod, DuckDbWorkerObjectSourceParams, DuckDbWorkerRequest,
DuckDbWorkerResponse, DuckDbWorkerTableParams,
};
use crate::models::connection::AttachedDatabaseConfig;
use crate::path_utils::expand_tilde;
#[derive(Default)]
pub struct DuckDbWorkerSession {
connection: Option<Arc<db::duckdb_driver::DuckDbConnection>>,
attached_names: Vec<String>,
}
impl DuckDbWorkerSession {
pub fn connect(&mut self, params: DuckDbWorkerConnectParams) -> Result<(), String> {
let path = expand_tilde(&params.path);
let connection = db::duckdb_driver::connect_path(&path)?;
let mut attached_names = Vec::new();
{
let locked = connection.lock().map_err(|e| e.to_string())?;
for attached in &params.attached_databases {
let path = expand_tilde(&attached.path);
crate::schema::duckdb_attach_database(&locked, &attached.name, &path)?;
attached_names.push(attached.name.clone());
}
if let Some(script) = params.init_script.as_deref() {
// Track script attachments by diffing the actual catalog list
// instead of parsing the SQL, so ATTACH without an alias,
// dynamic statements, and extension-specific syntax all count.
let before = duckdb_catalog_list(&locked)?;
db::duckdb_driver::run_init_script(&locked, script)?;
for name in duckdb_catalog_list(&locked)? {
if !before.iter().any(|existing| existing.eq_ignore_ascii_case(&name))
&& !attached_names.iter().any(|attached| attached.eq_ignore_ascii_case(&name))
{
attached_names.push(name);
}
}
}
}
self.connection = Some(connection);
self.attached_names = attached_names;
Ok(())
}
pub fn execute(&mut self, params: DuckDbWorkerExecuteParams) -> Result<db::QueryResult, String> {
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
let locked = connection.lock().map_err(|e| e.to_string())?;
let result = crate::query::duckdb_execute_for_database(
&locked,
&self.attached_names,
params.database.as_deref(),
&params.sql,
params.max_rows,
)?;
if let Some(name) = crate::db::duckdb_sql::attached_name_from_attach_sql(&params.sql) {
if !self.attached_names.iter().any(|attached| attached.eq_ignore_ascii_case(&name)) {
self.attached_names.push(name);
}
}
Ok(result)
}
pub fn list_databases(&self) -> Result<Vec<db::DatabaseInfo>, String> {
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
let locked = connection.lock().map_err(|e| e.to_string())?;
crate::schema::duckdb_list_databases_with_attached(&locked, &self.attached_names)
}
pub fn list_schemas(&self, params: DuckDbWorkerDatabaseParams) -> Result<Vec<String>, String> {
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
let locked = connection.lock().map_err(|e| e.to_string())?;
crate::schema::duckdb_list_schemas_with_attached(&locked, &params.database, &self.attached_names)
}
pub fn list_tables(&self, params: DuckDbWorkerTableParams) -> Result<Vec<db::TableInfo>, String> {
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
let locked = connection.lock().map_err(|e| e.to_string())?;
crate::schema::duckdb_query_tables_in_database_with_attached(
&locked,
&params.database,
&params.schema,
&self.attached_names,
)
}
pub fn list_columns(&self, params: DuckDbWorkerColumnParams) -> Result<Vec<db::ColumnInfo>, String> {
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
let locked = connection.lock().map_err(|e| e.to_string())?;
crate::schema::duckdb_query_columns_in_database_with_attached(
&locked,
&params.database,
&params.schema,
&params.table,
&self.attached_names,
)
}
pub fn get_object_source(&self, params: DuckDbWorkerObjectSourceParams) -> Result<String, String> {
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
let locked = connection.lock().map_err(|e| e.to_string())?;
crate::schema::duckdb_object_source_with_attached(
&locked,
&params.database,
&params.schema,
&params.name,
&params.object_type,
&self.attached_names,
)
}
pub fn attach_database(&mut self, attached: AttachedDatabaseConfig) -> Result<(), String> {
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
let locked = connection.lock().map_err(|e| e.to_string())?;
let path = expand_tilde(&attached.path);
crate::schema::duckdb_attach_database(&locked, &attached.name, &path)?;
self.attached_names.push(attached.name);
Ok(())
}
fn interrupt_handle(&self) -> Result<Arc<duckdb::InterruptHandle>, String> {
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
Ok(connection.interrupt_handle())
}
/// Probes whether the connection is still usable after an execute error.
///
/// duckdb-rs 1.10503.1 has a bug where `Connection::prepare()` failing at the
/// `duckdb_extract_statements` stage (a syntax `Parser Error`) permanently poisons
/// the connection: every later operation returns `resource deadlock would occur`,
/// and dropping the connection aborts the whole process. Benign errors (binder,
/// catalog, runtime) do not poison the connection.
///
/// This probe runs a trivial query through `execute_batch`, which uses the
/// non-poisoning `duckdb_query_arrow` path and fails fast on a poisoned connection.
/// It returns `true` when the connection is healthy and safe to reuse.
fn is_connection_healthy(&self) -> bool {
match self.connection.as_ref() {
Some(connection) => match connection.lock() {
Ok(locked) => locked.execute_batch("SELECT 1").is_ok(),
Err(_) => false,
},
None => false,
}
}
}
fn duckdb_catalog_list(con: &duckdb::Connection) -> Result<Vec<String>, String> {
let mut stmt = con.prepare("SHOW DATABASES").map_err(|e| e.to_string())?;
let rows = stmt.query_map([], |row| row.get::<_, String>(0)).map_err(|e| e.to_string())?;
Ok(rows.filter_map(|row| row.ok()).collect())
}
#[derive(Clone, Default)]
struct DuckDbWorkerRuntime {
session: Arc<Mutex<DuckDbWorkerSession>>,
active_interrupt: Arc<Mutex<Option<Arc<duckdb::InterruptHandle>>>>,
}
struct WorkerHandleResult {
response: Option<DuckDbWorkerResponse>,
shutdown: bool,
}
impl DuckDbWorkerRuntime {
async fn handle_request(
&self,
request: DuckDbWorkerRequest,
stdout: Arc<tokio::sync::Mutex<Stdout>>,
) -> WorkerHandleResult {
match request.method {
DuckDbWorkerMethod::Connect => self.handle_session_request(request, |session, request| {
session.connect(request.parse_params()?)?;
Ok(DuckDbWorkerResponse::ok(request.id, serde_json::json!({ "connected": true })))
}),
DuckDbWorkerMethod::ListDatabases => self.handle_session_request(request, |session, request| {
Ok(DuckDbWorkerResponse::ok(request.id, session.list_databases()?))
}),
DuckDbWorkerMethod::ListSchemas => self.handle_session_request(request, |session, request| {
let params = request.parse_params()?;
Ok(DuckDbWorkerResponse::ok(request.id, session.list_schemas(params)?))
}),
DuckDbWorkerMethod::ListTables => self.handle_session_request(request, |session, request| {
let params = request.parse_params()?;
Ok(DuckDbWorkerResponse::ok(request.id, session.list_tables(params)?))
}),
DuckDbWorkerMethod::ListColumns => self.handle_session_request(request, |session, request| {
let params = request.parse_params()?;
Ok(DuckDbWorkerResponse::ok(request.id, session.list_columns(params)?))
}),
DuckDbWorkerMethod::GetObjectSource => self.handle_session_request(request, |session, request| {
let params = request.parse_params()?;
Ok(DuckDbWorkerResponse::ok(request.id, session.get_object_source(params)?))
}),
DuckDbWorkerMethod::AttachDatabase => self.handle_session_request(request, |session, request| {
session.attach_database(request.parse_params()?)?;
Ok(DuckDbWorkerResponse::ok_empty(request.id))
}),
DuckDbWorkerMethod::Execute => {
if self.active_interrupt.lock().unwrap_or_else(|e| e.into_inner()).is_some() {
return WorkerHandleResult {
response: Some(DuckDbWorkerResponse::err(
request.id,
DuckDbWorkerError::new("duckdb_worker_busy", "DuckDB worker is already executing a query"),
)),
shutdown: false,
};
}
let interrupt_handle = match self.session.lock().unwrap_or_else(|e| e.into_inner()).interrupt_handle() {
Ok(handle) => handle,
Err(err) => {
return WorkerHandleResult {
response: Some(DuckDbWorkerResponse::err(
request.id,
DuckDbWorkerError::from_message("duckdb_not_connected", err),
)),
shutdown: false,
}
}
};
*self.active_interrupt.lock().unwrap_or_else(|e| e.into_inner()) = Some(interrupt_handle);
let session = self.session.clone();
let active_interrupt = self.active_interrupt.clone();
tokio::spawn(async move {
let id = request.id.clone();
let params = request.parse_params::<DuckDbWorkerExecuteParams>();
let (result, poisoned) = match params {
Ok(params) => {
let probe_session = session.clone();
tokio::task::spawn_blocking(move || {
let mut session = probe_session.lock().unwrap_or_else(|e| e.into_inner());
let result = session.execute(params);
// Only probe connection health when execute failed: a syntax
// Parser Error can poison the connection (duckdb-rs bug), and any
// later use — including drop — would abort the process.
let poisoned = result.is_err() && !session.is_connection_healthy();
(result, poisoned)
})
.await
.map_err(|e| e.to_string())
.unwrap_or_else(|err| (Err(err), true))
}
Err(err) => (Err(err.message), false),
};
*active_interrupt.lock().unwrap_or_else(|e| e.into_inner()) = None;
let response = match result {
Ok(result) => DuckDbWorkerResponse::ok(id, result),
// A poisoned connection cannot be reused or even safely dropped in-process.
// Report a distinct code so the parent kills this worker (OS-level, skipping
// destructors) and restarts a fresh one on the next request. We must not exit
// here ourselves: process::exit races the parent's next request, which could
// be written to a dying worker. Killing on the parent side has no such window.
Err(err) if poisoned => {
log::warn!("[duckdb-worker:poisoned-connection] reporting to parent for restart");
DuckDbWorkerResponse::err(
id,
DuckDbWorkerError::from_message("duckdb_worker_poisoned", err),
)
}
Err(err) => {
DuckDbWorkerResponse::err(id, DuckDbWorkerError::from_message("duckdb_execute_failed", err))
}
};
write_response(stdout, &response).await;
});
WorkerHandleResult { response: None, shutdown: false }
}
DuckDbWorkerMethod::Cancel => {
if let Some(handle) = self.active_interrupt.lock().unwrap_or_else(|e| e.into_inner()).as_ref() {
handle.interrupt();
}
WorkerHandleResult {
response: Some(DuckDbWorkerResponse::ok(request.id, serde_json::json!({ "cancelled": true }))),
shutdown: false,
}
}
DuckDbWorkerMethod::Shutdown => WorkerHandleResult {
response: Some(DuckDbWorkerResponse::ok(request.id, serde_json::json!({ "shutdown": true }))),
shutdown: true,
},
}
}
fn handle_session_request(
&self,
request: DuckDbWorkerRequest,
handler: impl FnOnce(
&mut DuckDbWorkerSession,
DuckDbWorkerRequest,
) -> Result<DuckDbWorkerResponse, DuckDbWorkerError>,
) -> WorkerHandleResult {
let id = request.id.clone();
if self.active_interrupt.lock().unwrap_or_else(|e| e.into_inner()).is_some() {
return WorkerHandleResult {
response: Some(DuckDbWorkerResponse::err(
id,
DuckDbWorkerError::new("duckdb_worker_busy", "DuckDB worker is executing a query"),
)),
shutdown: false,
};
}
let response = {
match self.session.try_lock() {
Ok(mut session) => match handler(&mut session, request) {
Ok(response) => response,
Err(err) => DuckDbWorkerResponse::err(id, err),
},
Err(_) => DuckDbWorkerResponse::err(
id,
DuckDbWorkerError::new("duckdb_worker_busy", "DuckDB worker session is busy"),
),
}
};
WorkerHandleResult { response: Some(response), shutdown: false }
}
}
pub async fn run_stdio_worker() -> Result<(), String> {
let runtime = DuckDbWorkerRuntime::default();
let stdout = Arc::new(tokio::sync::Mutex::new(tokio::io::stdout()));
let stdin = BufReader::new(tokio::io::stdin());
let mut lines = stdin.lines();
loop {
let line = lines.next_line().await.map_err(|e| e.to_string())?;
let Some(line) = line else {
std::process::exit(0);
};
if line.trim().is_empty() {
continue;
}
let request: DuckDbWorkerRequest = match serde_json::from_str(&line) {
Ok(request) => request,
Err(err) => {
let response = DuckDbWorkerResponse::err(
"",
DuckDbWorkerError::new("invalid_json", format!("Invalid worker request JSON: {err}")),
);
write_response(stdout.clone(), &response).await;
continue;
}
};
let result = runtime.handle_request(request, stdout.clone()).await;
if let Some(response) = result.response {
write_response(stdout.clone(), &response).await;
}
if result.shutdown {
break;
}
}
Ok(())
}
async fn write_response(stdout: Arc<tokio::sync::Mutex<Stdout>>, response: &DuckDbWorkerResponse) {
let Ok(line) = serde_json::to_string(response) else {
return;
};
let mut stdout = stdout.lock().await;
let _ = stdout.write_all(line.as_bytes()).await;
let _ = stdout.write_all(b"\n").await;
let _ = stdout.flush().await;
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn worker_session_connects_to_memory_duckdb() {
let mut session = DuckDbWorkerSession::default();
session
.connect(DuckDbWorkerConnectParams {
path: ":memory:".to_string(),
attached_databases: Vec::new(),
init_script: None,
})
.expect("connect");
assert_eq!(session.list_databases().expect("list databases")[0].name, "main");
}
#[test]
fn worker_session_executes_select_query() {
let mut session = DuckDbWorkerSession::default();
session
.connect(DuckDbWorkerConnectParams {
path: ":memory:".to_string(),
attached_databases: Vec::new(),
init_script: None,
})
.expect("connect");
let result = session
.execute(DuckDbWorkerExecuteParams { sql: "SELECT 1 AS value".to_string(), database: None, max_rows: None })
.expect("execute");
assert_eq!(result.columns, vec!["value"]);
assert_eq!(result.rows, vec![vec![serde_json::json!(1)]]);
}
#[test]
fn worker_session_lists_tables_and_columns() {
let mut session = DuckDbWorkerSession::default();
session
.connect(DuckDbWorkerConnectParams {
path: ":memory:".to_string(),
attached_databases: Vec::new(),
init_script: None,
})
.expect("connect");
session
.execute(DuckDbWorkerExecuteParams {
sql: "CREATE TABLE users (id INTEGER PRIMARY KEY, name VARCHAR)".to_string(),
database: None,
max_rows: None,
})
.expect("create table");
let schemas =
session.list_schemas(DuckDbWorkerDatabaseParams { database: "main".to_string() }).expect("list schemas");
let tables = session
.list_tables(DuckDbWorkerTableParams { database: "main".to_string(), schema: "main".to_string() })
.expect("list tables");
let columns = session
.list_columns(DuckDbWorkerColumnParams {
database: "main".to_string(),
schema: "main".to_string(),
table: "users".to_string(),
})
.expect("list columns");
assert!(schemas.iter().any(|schema| schema == "main"));
assert!(tables.iter().any(|table| table.name == "users"));
assert!(columns.iter().any(|column| column.name == "id" && column.is_primary_key));
}
#[test]
fn worker_session_reads_view_source() {
let mut session = DuckDbWorkerSession::default();
session
.connect(DuckDbWorkerConnectParams {
path: ":memory:".to_string(),
attached_databases: Vec::new(),
init_script: None,
})
.expect("connect");
session
.execute(DuckDbWorkerExecuteParams {
sql: "CREATE VIEW active_orders AS SELECT 1 AS id".to_string(),
database: None,
max_rows: None,
})
.expect("create view");
let source = session
.get_object_source(DuckDbWorkerObjectSourceParams {
database: "main".to_string(),
schema: "main".to_string(),
name: "active_orders".to_string(),
object_type: db::ObjectSourceKind::View,
})
.expect("get view source");
assert!(source.starts_with("CREATE VIEW"));
assert!(source.contains("active_orders"));
}
#[test]
fn worker_session_attaches_database() {
let dir = std::env::temp_dir().join(format!("dbx-duckdb-worker-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).expect("create temp dir");
let attached_path = dir.join("analytics.duckdb");
{
let con = duckdb::Connection::open(&attached_path).expect("create attached db");
con.execute_batch("CREATE TABLE events (id INTEGER);").expect("create attached table");
}
let mut session = DuckDbWorkerSession::default();
session
.connect(DuckDbWorkerConnectParams {
path: ":memory:".to_string(),
attached_databases: Vec::new(),
init_script: None,
})
.expect("connect");
session
.attach_database(AttachedDatabaseConfig {
name: "analytics".to_string(),
path: attached_path.to_string_lossy().to_string(),
})
.expect("attach");
let databases = session.list_databases().expect("list databases");
let tables = session
.list_tables(DuckDbWorkerTableParams { database: "analytics".to_string(), schema: "main".to_string() })
.expect("list tables");
assert!(databases.iter().any(|database| database.name == "analytics"));
assert!(tables.iter().any(|table| table.name == "events"));
let _ = std::fs::remove_dir_all(dir);
}
#[test]
fn worker_session_runs_init_script_and_tracks_attached_alias() {
let dir = std::env::temp_dir().join(format!("dbx-duckdb-worker-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).expect("create temp dir");
let attached_path = dir.join("warehouse.duckdb");
{
let con = duckdb::Connection::open(&attached_path).expect("create attached db");
con.execute_batch("CREATE TABLE facts (id INTEGER);").expect("create attached table");
}
let mut session = DuckDbWorkerSession::default();
let script = format!(
"SET memory_limit='512MB'; -- session tuning\nATTACH '{}' AS warehouse;\nCREATE TABLE staging (id INTEGER);",
attached_path.to_string_lossy().replace('\'', "''")
);
session
.connect(DuckDbWorkerConnectParams {
path: ":memory:".to_string(),
attached_databases: Vec::new(),
init_script: Some(script),
})
.expect("connect");
assert!(session.attached_names.iter().any(|name| name == "warehouse"));
let databases = session.list_databases().expect("list databases");
assert!(databases.iter().any(|database| database.name == "warehouse"));
let tables = session
.list_tables(DuckDbWorkerTableParams { database: "warehouse".to_string(), schema: "main".to_string() })
.expect("list tables");
assert!(tables.iter().any(|table| table.name == "facts"));
let _ = std::fs::remove_dir_all(dir);
}
#[test]
fn worker_session_tracks_init_script_attach_without_alias() {
let dir = std::env::temp_dir().join(format!("dbx-duckdb-worker-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).expect("create temp dir");
let attached_path = dir.join("noalias.duckdb");
{
let con = duckdb::Connection::open(&attached_path).expect("create attached db");
con.execute_batch("CREATE TABLE items (id INTEGER);").expect("create attached table");
}
let mut session = DuckDbWorkerSession::default();
session
.connect(DuckDbWorkerConnectParams {
path: ":memory:".to_string(),
attached_databases: Vec::new(),
init_script: Some(format!("ATTACH '{}';", attached_path.to_string_lossy().replace('\'', "''"))),
})
.expect("connect");
assert!(session.attached_names.iter().any(|name| name == "noalias"), "names: {:?}", session.attached_names);
let tables = session
.list_tables(DuckDbWorkerTableParams { database: "noalias".to_string(), schema: "main".to_string() })
.expect("list tables");
assert!(tables.iter().any(|table| table.name == "items"));
let _ = std::fs::remove_dir_all(dir);
}
#[test]
fn worker_session_reports_failing_init_script_statement() {
let mut session = DuckDbWorkerSession::default();
let err = session
.connect(DuckDbWorkerConnectParams {
path: ":memory:".to_string(),
attached_databases: Vec::new(),
init_script: Some("SELECT 1; SELECT * FROM missing_table;".to_string()),
})
.expect_err("init script should fail");
assert!(err.contains("statement 2"), "unexpected error: {err}");
}
#[test]
fn worker_session_tracks_attach_sql_alias() {
let dir = std::env::temp_dir().join(format!("dbx-duckdb-worker-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).expect("create temp dir");
let attached_path = dir.join("sales.duckdb");
{
let con = duckdb::Connection::open(&attached_path).expect("create attached db");
con.execute_batch("CREATE TABLE orders (id INTEGER);").expect("create attached table");
}
let mut session = DuckDbWorkerSession::default();
session
.connect(DuckDbWorkerConnectParams {
path: ":memory:".to_string(),
attached_databases: Vec::new(),
init_script: None,
})
.expect("connect");
session
.execute(DuckDbWorkerExecuteParams {
sql: format!("ATTACH '{}' AS \"sales db\";", attached_path.to_string_lossy().replace('\'', "''")),
database: None,
max_rows: None,
})
.expect("attach sql");
assert!(session.attached_names.iter().any(|name| name == "sales db"));
let tables = session
.list_tables(DuckDbWorkerTableParams { database: "sales db".to_string(), schema: "main".to_string() })
.expect("list tables");
assert!(tables.iter().any(|table| table.name == "orders"));
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn metadata_returns_busy_while_query_is_active() {
let runtime = DuckDbWorkerRuntime::default();
let stdout = Arc::new(tokio::sync::Mutex::new(tokio::io::stdout()));
*runtime.active_interrupt.lock().unwrap_or_else(|e| e.into_inner()) =
Some(duckdb::Connection::open_in_memory().expect("connection").interrupt_handle());
let result = runtime
.handle_request(
DuckDbWorkerRequest::new("req-busy", DuckDbWorkerMethod::ListDatabases, serde_json::json!({}))
.expect("request"),
stdout,
)
.await;
let response = result.response.expect("response");
assert!(!response.ok);
assert_eq!(response.error.expect("error").code, "duckdb_worker_busy");
}
}
+2 -5
View File
@@ -5,14 +5,11 @@ pub mod cloudflare_d1;
pub use cloudflare_d1 as cloudflare_d1_driver;
pub mod document_result;
pub mod doris;
pub mod duckdb_driver;
pub mod duckdb_sql;
#[cfg(feature = "duckdb-bundled")]
#[cfg(feature = "duckdb-sidecar")]
pub mod duckdb_worker_process;
#[cfg(feature = "duckdb-bundled")]
#[cfg(feature = "duckdb-sidecar")]
pub mod duckdb_worker_protocol;
#[cfg(feature = "duckdb-bundled")]
pub mod duckdb_worker_runtime;
pub mod elasticsearch_driver;
pub mod elasticsearch_sql;
pub mod file_validator;
+3 -3
View File
@@ -63,7 +63,7 @@ pub enum DatabaseCreationTarget {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
#[cfg(feature = "duckdb-bundled")]
#[cfg(feature = "duckdb-sidecar")]
pub struct DuckDbAttachDatabaseSqlOptions {
pub path: String,
pub name: String,
@@ -312,7 +312,7 @@ pub fn supports_database_property_comment(database_type: Option<DatabaseType>) -
)
}
#[cfg(feature = "duckdb-bundled")]
#[cfg(feature = "duckdb-sidecar")]
pub fn build_duckdb_attach_database_sql(options: DuckDbAttachDatabaseSqlOptions) -> String {
format!(
"ATTACH {} AS {};",
@@ -1096,7 +1096,7 @@ mod tests {
assert!(!supports_create_database_charset(Some(DatabaseType::Postgres), None));
}
#[cfg(feature = "duckdb-bundled")]
#[cfg(feature = "duckdb-sidecar")]
#[test]
fn builds_duckdb_attach_sql() {
assert_eq!(
-173
View File
@@ -1,173 +0,0 @@
#![cfg(feature = "duckdb-bundled")]
use std::sync::Arc;
use super::types::{CacheState, ExternalTableRef, ExternalTableSnapshot};
/// Loads an external table snapshot into an in-memory DuckDB connection.
pub fn load_snapshot_to_duckdb(con: &duckdb::Connection, snapshot: &ExternalTableSnapshot) -> Result<(), String> {
let table_name = sanitize_table_name(&snapshot.table_ref.table_name);
con.execute(&format!("DROP TABLE IF EXISTS \"{}\"", table_name), [])
.map_err(|e| format!("Failed to drop table: {e}"))?;
let col_defs: Vec<String> = snapshot
.columns
.iter()
.map(|c| {
format!(
"\"{}\" {}{}",
c.name.replace('"', "\"\""),
c.duckdb_type,
if c.is_nullable { "" } else { " NOT NULL" }
)
})
.collect();
let pk_cols: Vec<&str> = snapshot.columns.iter().filter(|c| c.is_primary_key).map(|c| c.name.as_str()).collect();
let mut create_sql = format!("CREATE TABLE \"{}\" ({}", table_name, col_defs.join(", "));
if !pk_cols.is_empty() {
create_sql.push_str(&format!(
", PRIMARY KEY ({})",
pk_cols.iter().map(|c| format!("\"{}\"", c.replace('"', "\"\""))).collect::<Vec<_>>().join(", ")
));
}
create_sql.push(')');
con.execute(&create_sql, []).map_err(|e| format!("Failed to create table: {e}"))?;
if snapshot.rows.is_empty() {
return Ok(());
}
let placeholders: Vec<&str> = (0..snapshot.columns.len()).map(|_| "?").collect();
let insert_sql = format!("INSERT INTO \"{}\" VALUES ({})", table_name, placeholders.join(", "));
let mut stmt = con.prepare(&insert_sql).map_err(|e| format!("Failed to prepare insert: {e}"))?;
for row in &snapshot.rows {
let params: Vec<Box<dyn duckdb::ToSql>> = row
.iter()
.enumerate()
.map(|(i, val)| {
let col_type = snapshot.columns.get(i).map(|c| c.duckdb_type.as_str()).unwrap_or("VARCHAR");
json_to_duckdb_param(val, col_type)
})
.collect();
let param_refs: Vec<&dyn duckdb::ToSql> = params.iter().map(|p| p.as_ref()).collect();
stmt.execute(param_refs.as_slice()).map_err(|e| format!("Failed to insert row: {e}"))?;
}
Ok(())
}
fn json_to_duckdb_param(val: &serde_json::Value, col_type: &str) -> Box<dyn duckdb::ToSql> {
match val {
serde_json::Value::Null => Box::new(None::<String>),
serde_json::Value::Bool(b) => Box::new(*b),
serde_json::Value::Number(n) => {
if col_type.contains("INT") {
Box::new(n.as_i64().unwrap_or(0))
} else if col_type.contains("DOUBLE") || col_type.contains("FLOAT") || col_type.contains("DECIMAL") {
Box::new(n.as_f64().unwrap_or(0.0))
} else {
Box::new(n.to_string())
}
}
serde_json::Value::String(s) => Box::new(s.clone()),
_ => Box::new(val.to_string()),
}
}
/// Sanitize a source table name into a stable DuckDB table identifier.
pub fn sanitize_table_name(name: &str) -> String {
let sanitized: String = name.chars().map(|c| if c.is_alphanumeric() || c == '_' { c } else { '_' }).collect();
if sanitized.is_empty() {
"_unnamed_".to_string()
} else if sanitized.chars().next().is_some_and(|c| c.is_ascii_digit()) {
format!("_{sanitized}")
} else {
sanitized
}
}
/// Manages a DuckDB cache for one external source.
pub struct ExternalPool {
pub source: Arc<dyn super::traits::ExternalTabularSource>,
pub cache: Arc<std::sync::Mutex<duckdb::Connection>>,
pub cache_state: std::sync::Mutex<CacheState>,
pub table_map: std::sync::Mutex<std::collections::HashMap<String, ExternalTableRef>>,
}
impl ExternalPool {
pub fn new(
source: Arc<dyn super::traits::ExternalTabularSource>,
cache: Arc<std::sync::Mutex<duckdb::Connection>>,
) -> Self {
Self {
source,
cache,
cache_state: std::sync::Mutex::new(CacheState::Empty),
table_map: std::sync::Mutex::new(std::collections::HashMap::new()),
}
}
pub async fn refresh_cache(&self) -> Result<(), String> {
{
let mut state = self.cache_state.lock().map_err(|e| e.to_string())?;
*state = CacheState::Loading;
}
let tables = self.source.list_tables().await?;
let mut new_table_map = std::collections::HashMap::new();
for table_ref in &tables {
let snapshot = self.source.load_table(table_ref).await?;
let mut sanitized_name = sanitize_table_name(&table_ref.table_name);
if new_table_map.contains_key(&sanitized_name) {
let base = sanitized_name.clone();
let mut suffix = 2;
loop {
sanitized_name = format!("{base}_{suffix}");
if !new_table_map.contains_key(&sanitized_name) {
break;
}
suffix += 1;
}
}
let cache = self.cache.clone();
tokio::task::spawn_blocking(move || {
let con = cache.lock().map_err(|e| e.to_string())?;
load_snapshot_to_duckdb(&con, &snapshot)
})
.await
.map_err(|e| e.to_string())??;
new_table_map.insert(sanitized_name, table_ref.clone());
}
{
let mut map = self.table_map.lock().map_err(|e| e.to_string())?;
*map = new_table_map;
}
{
let mut state = self.cache_state.lock().map_err(|e| e.to_string())?;
*state = CacheState::Fresh;
}
Ok(())
}
}
impl std::fmt::Debug for ExternalPool {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ExternalPool")
.field("source", &self.source.display_name())
.field("cache_state", &self.cache_state)
.finish()
}
}
-3
View File
@@ -1,8 +1,5 @@
pub mod duckdb_cache;
pub mod traits;
pub mod types;
#[cfg(feature = "duckdb-bundled")]
pub use duckdb_cache::ExternalPool;
pub use traits::ExternalTabularSource;
pub use types::*;
+5 -825
View File
@@ -1,7 +1,3 @@
#[cfg(feature = "duckdb-bundled")]
use chrono::{DateTime, Duration as ChronoDuration, NaiveDate, NaiveDateTime, NaiveTime, Utc};
#[cfg(feature = "duckdb-bundled")]
use duckdb::types::{TimeUnit, Value, ValueRef};
use futures::StreamExt;
use mysql_async::prelude::Queryable;
use serde::{Deserialize, Serialize};
@@ -18,10 +14,6 @@ use std::sync::{
Arc,
};
use std::time::Duration;
#[cfg(feature = "duckdb-bundled")]
use tokio::task::JoinHandle;
#[cfg(feature = "duckdb-bundled")]
use tokio::time::sleep;
use tokio::time::timeout;
use tokio_util::sync::CancellationToken;
@@ -30,8 +22,6 @@ use crate::database_capabilities;
use crate::db;
use crate::models::connection::{ConnectionConfig, DatabaseType};
use crate::query_execution_sql::is_write_sql;
#[cfg(feature = "duckdb-bundled")]
use crate::sql::starts_with_duckdb_result_sql_keyword;
use crate::sql::{split_sql_batches, split_sql_statements, starts_with_executable_sql_keyword};
use crate::sql_dialect::{resolve_for_db, CAP_TRANSACTIONAL_DDL};
use crate::sql_risk::{classify_sql_risk_for_database, SqlRisk};
@@ -45,10 +35,6 @@ pub const QUERY_CANCELED: &str = "Query canceled";
const MONGO_SHELL_COMMAND_HINT: &str = "Use MongoDB shell-style commands, for example: db.collection.find({}).limit(100), db.collection.aggregate([]), db.collection.aggregate([], { explain: true }), db.version(), db.collection.countDocuments({}), db.collection.distinct(\"field\"), db.collection.getIndexes(), db.collection.createIndex({...}), or db.collection.insertOne({...}).";
const SQL_OMITTED_ERROR_CONTEXT: &str =
"SQL text omitted from user-facing error; enable debug SQL diagnostics for a redacted statement.";
#[cfg(feature = "duckdb-bundled")]
const DUCKDB_INTERRUPT_DRAIN_TIMEOUT: Duration = Duration::from_secs(2);
#[cfg(feature = "duckdb-bundled")]
const DUCKDB_DRAINING_MESSAGE: &str = "The previous DuckDB query is still stopping. Please try again shortly.";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PoolErrorAction {
@@ -476,387 +462,6 @@ fn query_result_row_limit(max_rows: Option<usize>) -> usize {
max_rows.unwrap_or(MAX_ROWS).max(1)
}
#[cfg(feature = "duckdb-bundled")]
pub fn duckdb_execute(con: &duckdb::Connection, sql: &str) -> Result<db::QueryResult, String> {
duckdb_execute_with_max_rows(con, sql, None)
}
#[cfg(feature = "duckdb-bundled")]
fn duckdb_value_to_json(row: &duckdb::Row<'_>, idx: usize) -> serde_json::Value {
let Ok(value_ref) = row.get_ref(idx) else {
return serde_json::Value::Null;
};
match value_ref {
ValueRef::Null => serde_json::Value::Null,
ValueRef::Boolean(b) => serde_json::Value::Bool(b),
ValueRef::TinyInt(i) => serde_json::Value::Number((i as i64).into()),
ValueRef::SmallInt(i) => serde_json::Value::Number((i as i64).into()),
ValueRef::Int(i) => serde_json::Value::Number((i as i64).into()),
ValueRef::BigInt(i) => serde_json::Value::Number(i.into()),
ValueRef::HugeInt(i) => serde_json::Value::String(i.to_string()),
ValueRef::UTinyInt(i) => serde_json::Value::Number((i as u64).into()),
ValueRef::USmallInt(i) => serde_json::Value::Number((i as u64).into()),
ValueRef::UInt(i) => serde_json::Value::Number((i as u64).into()),
ValueRef::UBigInt(i) => serde_json::Value::Number(i.into()),
ValueRef::Float(f) => {
serde_json::Number::from_f64(f as f64).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null)
}
ValueRef::Double(f) => {
serde_json::Number::from_f64(f).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null)
}
ValueRef::Decimal(d) => serde_json::Value::String(d.to_string()),
ValueRef::Date32(days) => {
duckdb_date32_to_string(days).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
}
ValueRef::Time64(unit, value) => {
duckdb_time64_to_string(unit, value).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
}
ValueRef::Timestamp(unit, value) => {
duckdb_timestamp_to_string(unit, value).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
}
ValueRef::Text(bytes) => std::str::from_utf8(bytes)
.map(|s| serde_json::Value::String(s.to_string()))
.unwrap_or(serde_json::Value::Null),
ValueRef::Blob(bytes) => {
let hex: String = bytes.iter().map(|b| format!("{:02x}", b)).collect();
serde_json::Value::String(format!("\\x{hex}"))
}
ValueRef::Interval { months, days, nanos } => {
serde_json::Value::String(duckdb_interval_to_string(months, days, nanos))
}
ValueRef::List(..)
| ValueRef::Array(..)
| ValueRef::Struct(..)
| ValueRef::Map(..)
| ValueRef::Enum(..)
| ValueRef::Union(..) => duckdb_owned_value_to_json(&value_ref.to_owned()),
_ => duckdb_owned_value_to_json(&value_ref.to_owned()),
}
}
#[cfg(feature = "duckdb-bundled")]
fn duckdb_owned_value_to_json(value: &Value) -> serde_json::Value {
match value {
Value::Null => serde_json::Value::Null,
Value::Boolean(b) => serde_json::Value::Bool(*b),
Value::TinyInt(i) => serde_json::Value::Number((*i as i64).into()),
Value::SmallInt(i) => serde_json::Value::Number((*i as i64).into()),
Value::Int(i) => serde_json::Value::Number((*i as i64).into()),
Value::BigInt(i) => serde_json::Value::Number((*i).into()),
Value::HugeInt(i) => serde_json::Value::String(i.to_string()),
Value::UTinyInt(i) => serde_json::Value::Number((*i as u64).into()),
Value::USmallInt(i) => serde_json::Value::Number((*i as u64).into()),
Value::UInt(i) => serde_json::Value::Number((*i as u64).into()),
Value::UBigInt(i) => serde_json::Value::Number((*i).into()),
Value::Float(f) => {
serde_json::Number::from_f64(*f as f64).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null)
}
Value::Double(f) => {
serde_json::Number::from_f64(*f).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null)
}
Value::Decimal(d) => serde_json::Value::String(d.to_string()),
Value::Timestamp(unit, value) => {
duckdb_timestamp_to_string(*unit, *value).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
}
Value::Text(text) | Value::Enum(text) => serde_json::Value::String(text.clone()),
Value::Blob(bytes) => {
let hex: String = bytes.iter().map(|b| format!("{:02x}", b)).collect();
serde_json::Value::String(format!("\\x{hex}"))
}
Value::Date32(days) => {
duckdb_date32_to_string(*days).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
}
Value::Time64(unit, value) => {
duckdb_time64_to_string(*unit, *value).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
}
Value::Interval { months, days, nanos } => {
serde_json::Value::String(duckdb_interval_to_string(*months, *days, *nanos))
}
Value::List(values) | Value::Array(values) => {
serde_json::Value::Array(values.iter().map(duckdb_owned_value_to_json).collect())
}
Value::Struct(entries) => serde_json::Value::Object(
entries.iter().map(|(key, value)| (key.clone(), duckdb_owned_value_to_json(value))).collect(),
),
Value::Map(entries) => serde_json::Value::Array(
entries
.iter()
.map(|(key, value)| {
serde_json::json!({
"key": duckdb_owned_value_to_json(key),
"value": duckdb_owned_value_to_json(value),
})
})
.collect(),
),
Value::Union(value) => duckdb_owned_value_to_json(value),
_ => serde_json::Value::Null,
}
}
#[cfg(feature = "duckdb-bundled")]
fn duckdb_interval_to_string(months: i32, days: i32, nanos: i64) -> String {
let mut parts = Vec::new();
if months != 0 {
let years = months / 12;
let rem = months % 12;
if years != 0 {
parts.push(format!("{} year{}", years, if years.abs() != 1 { "s" } else { "" }));
}
if rem != 0 {
parts.push(format!("{} mon{}", rem, if rem.abs() != 1 { "s" } else { "" }));
}
}
if days != 0 {
parts.push(format!("{} day{}", days, if days.abs() != 1 { "s" } else { "" }));
}
if nanos != 0 {
let total_secs = nanos / 1_000_000_000;
let hours = total_secs / 3600;
let mins = (total_secs % 3600) / 60;
let secs = total_secs % 60;
let sub_nanos = (nanos % 1_000_000_000).unsigned_abs();
if sub_nanos > 0 {
parts.push(format!(
"{:02}:{:02}:{:02}.{}",
hours,
mins,
secs,
format_temporal_without_empty_fraction(format!("0.{:09}", sub_nanos)).trim_start_matches("0.")
));
} else {
parts.push(format!("{:02}:{:02}:{:02}", hours, mins, secs));
}
}
if parts.is_empty() {
"00:00:00".to_string()
} else {
parts.join(" ")
}
}
#[cfg(feature = "duckdb-bundled")]
fn duckdb_date32_to_string(days: i32) -> Option<String> {
let epoch = NaiveDate::from_ymd_opt(1970, 1, 1)?;
epoch.checked_add_signed(ChronoDuration::days(i64::from(days))).map(|date| date.to_string())
}
#[cfg(feature = "duckdb-bundled")]
fn duckdb_time64_to_string(unit: TimeUnit, value: i64) -> Option<String> {
let nanos = duckdb_time_unit_to_nanos(unit, value)?;
let seconds = nanos.div_euclid(1_000_000_000);
let nanos_remainder = nanos.rem_euclid(1_000_000_000) as u32;
if !(0..86_400).contains(&seconds) {
return None;
}
let time = NaiveTime::from_num_seconds_from_midnight_opt(seconds as u32, nanos_remainder)?;
Some(format_temporal_without_empty_fraction(time.to_string()))
}
#[cfg(feature = "duckdb-bundled")]
fn duckdb_timestamp_to_string(unit: TimeUnit, value: i64) -> Option<String> {
let nanos = duckdb_time_unit_to_nanos(unit, value)?;
let seconds = nanos.div_euclid(1_000_000_000);
let nanos_remainder = nanos.rem_euclid(1_000_000_000) as u32;
let dt: DateTime<Utc> = DateTime::from_timestamp(seconds, nanos_remainder)?;
Some(format_naive_datetime(dt.naive_utc()))
}
#[cfg(feature = "duckdb-bundled")]
fn duckdb_time_unit_to_nanos(unit: TimeUnit, value: i64) -> Option<i64> {
match unit {
TimeUnit::Second => value.checked_mul(1_000_000_000),
TimeUnit::Millisecond => value.checked_mul(1_000_000),
TimeUnit::Microsecond => value.checked_mul(1_000),
TimeUnit::Nanosecond => Some(value),
}
}
#[cfg(feature = "duckdb-bundled")]
fn format_naive_datetime(value: NaiveDateTime) -> String {
if value.and_utc().timestamp_subsec_nanos() == 0 {
value.format("%Y-%m-%d %H:%M:%S").to_string()
} else {
format_temporal_without_empty_fraction(value.to_string())
}
}
#[cfg(feature = "duckdb-bundled")]
fn format_temporal_without_empty_fraction(value: String) -> String {
if !value.contains('.') {
return value;
}
let trimmed = value.trim_end_matches('0').trim_end_matches('.');
trimmed.to_string()
}
#[cfg(feature = "duckdb-bundled")]
pub fn duckdb_execute_with_max_rows(
con: &duckdb::Connection,
sql: &str,
max_rows: Option<usize>,
) -> Result<db::QueryResult, String> {
let start = std::time::Instant::now();
let row_limit = query_result_row_limit(max_rows);
if starts_with_duckdb_result_sql_keyword(sql) {
let mut stmt = con.prepare(sql).map_err(|e| e.to_string())?;
let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
let stmt_ref = rows.as_ref().ok_or("DuckDB statement unavailable")?;
let col_count = stmt_ref.column_count();
let columns: Vec<String> = (0..col_count)
.map(|i| stmt_ref.column_name(i).map(|s| s.to_string()).unwrap_or_else(|_| "?".to_string()))
.collect();
let mut result_rows = Vec::new();
while let Some(row) = rows.next().map_err(|e| e.to_string())? {
let vals: Vec<serde_json::Value> = (0..col_count).map(|i| duckdb_value_to_json(row, i)).collect();
result_rows.push(vals);
if result_rows.len() > row_limit {
break;
}
}
let truncated = result_rows.len() > row_limit;
if truncated {
result_rows.truncate(row_limit);
}
Ok(db::QueryResult {
columns,
column_types: Vec::new(),
column_sortables: vec![],
rows: result_rows,
affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(),
truncated,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
})
} else {
let affected = con.execute(sql, []).map_err(|e| e.to_string())?;
Ok(db::QueryResult {
columns: vec![],
column_types: Vec::new(),
column_sortables: vec![],
rows: vec![],
affected_rows: affected as u64,
execution_time_ms: start.elapsed().as_millis(),
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
})
}
}
#[cfg(feature = "duckdb-bundled")]
enum DuckDbTaskWait {
Finished(Result<db::QueryResult, String>),
Draining { error: String, task: JoinHandle<Result<db::QueryResult, String>> },
}
#[cfg(feature = "duckdb-bundled")]
fn duckdb_join_result(
result: Result<Result<db::QueryResult, String>, tokio::task::JoinError>,
) -> Result<db::QueryResult, String> {
result.map_err(|e| e.to_string())?
}
#[cfg(feature = "duckdb-bundled")]
async fn interrupt_and_drain_duckdb_task(
interrupt_handle: std::sync::Arc<duckdb::InterruptHandle>,
mut task: JoinHandle<Result<db::QueryResult, String>>,
error: String,
) -> DuckDbTaskWait {
interrupt_handle.interrupt();
match timeout(DUCKDB_INTERRUPT_DRAIN_TIMEOUT, &mut task).await {
Ok(result) => {
let _ = result;
DuckDbTaskWait::Finished(Err(error))
}
Err(_) => DuckDbTaskWait::Draining { error, task },
}
}
#[cfg(feature = "duckdb-bundled")]
async fn wait_for_duckdb_task_with_interrupt_outcome(
cancel_token: Option<CancellationToken>,
timeout_duration: Option<Duration>,
interrupt_handle: std::sync::Arc<duckdb::InterruptHandle>,
mut task: JoinHandle<Result<db::QueryResult, String>>,
) -> DuckDbTaskWait {
match (cancel_token, timeout_duration) {
(Some(token), Some(duration)) => {
tokio::select! {
biased;
_ = token.cancelled() => {
interrupt_and_drain_duckdb_task(interrupt_handle, task, canceled_error()).await
}
result = &mut task => DuckDbTaskWait::Finished(duckdb_join_result(result)),
_ = sleep(duration) => {
interrupt_and_drain_duckdb_task(interrupt_handle, task, timeout_error()).await
}
}
}
(Some(token), None) => {
tokio::select! {
biased;
_ = token.cancelled() => {
interrupt_and_drain_duckdb_task(interrupt_handle, task, canceled_error()).await
}
result = &mut task => DuckDbTaskWait::Finished(duckdb_join_result(result)),
}
}
(None, Some(duration)) => {
tokio::select! {
result = &mut task => DuckDbTaskWait::Finished(duckdb_join_result(result)),
_ = sleep(duration) => {
interrupt_and_drain_duckdb_task(interrupt_handle, task, timeout_error()).await
}
}
}
(None, None) => DuckDbTaskWait::Finished(duckdb_join_result(task.await)),
}
}
#[cfg(feature = "duckdb-bundled")]
async fn wait_for_duckdb_task_with_interrupt(
cancel_token: Option<CancellationToken>,
timeout_duration: Option<Duration>,
interrupt_handle: std::sync::Arc<duckdb::InterruptHandle>,
task: JoinHandle<Result<db::QueryResult, String>>,
) -> Result<db::QueryResult, String> {
match wait_for_duckdb_task_with_interrupt_outcome(cancel_token, timeout_duration, interrupt_handle, task).await {
DuckDbTaskWait::Finished(result) => result,
DuckDbTaskWait::Draining { error, .. } => Err(error),
}
}
#[cfg(feature = "duckdb-bundled")]
pub(crate) fn duckdb_execute_for_database(
con: &duckdb::Connection,
attached_names: &[String],
database: Option<&str>,
sql: &str,
max_rows: Option<usize>,
) -> Result<db::QueryResult, String> {
if let Some(database) = database.map(str::trim).filter(|database| !database.is_empty()) {
let catalog = if database == "main" {
crate::schema::duckdb_primary_catalog(con, attached_names)?
} else {
database.to_string()
};
con.execute_batch(&format!("USE {}", duckdb_quote_ident(&catalog))).map_err(|e| e.to_string())?;
}
duckdb_execute_with_max_rows(con, sql, max_rows)
}
#[cfg(feature = "duckdb-bundled")]
fn duckdb_quote_ident(value: &str) -> String {
format!("\"{}\"", value.replace('"', "\"\""))
}
pub fn truncate_result(result: db::QueryResult) -> db::QueryResult {
truncate_result_with_max_rows(result, None)
}
@@ -1166,11 +771,6 @@ where
}
}
#[cfg(feature = "duckdb-bundled")]
pub fn duckdb_draining_error() -> String {
DUCKDB_DRAINING_MESSAGE.to_string()
}
pub fn is_canceled(cancel_token: &Option<CancellationToken>) -> bool {
cancel_token.as_ref().map(|token| token.is_cancelled()).unwrap_or(false)
}
@@ -1366,12 +966,10 @@ pub async fn do_execute(
let _activity_touch = state.pool_activity_touch(pool_key);
let query_timeout = resolve_query_timeout(options.timeout_secs);
let (_duckdb_attached_names, read_only_connection) = {
let read_only_connection = {
let configs = state.configs.read().await;
let config = crate::connection::config_for_pool_key(pool_key, &configs);
let attached = config.map(crate::db::duckdb_sql::config_attached_names).unwrap_or_default();
let connection = config.filter(|config| config.read_only).map(|config| (config.name.clone(), config.db_type));
(attached, connection)
config.filter(|config| config.read_only).map(|config| (config.name.clone(), config.db_type))
};
let operation_budget = operation_budget_for_pool_key(state, pool_key, query_timeout).await;
if let Some((name, database_type)) = read_only_connection {
@@ -1383,49 +981,7 @@ pub async fn do_execute(
let pool = connections.get(pool_key).ok_or("Connection not found")?;
let result = match pool {
#[cfg(feature = "duckdb-bundled")]
PoolKind::DuckDb(con) => {
let con = con.clone();
if con.is_draining() {
drop(connections);
return Err(duckdb_draining_error());
}
let interrupt_handle = con.interrupt_handle();
if let Some(ref execution_id) = options.execution_id {
let cancel_interrupt_handle = interrupt_handle.clone();
state.running_queries.register_interrupt(execution_id, move || {
cancel_interrupt_handle.interrupt();
});
}
let sql = sql.to_string();
let database = database.map(str::to_string);
let attached_names = _duckdb_attached_names;
let max_rows = options.max_rows;
drop(connections);
let task_con = con.clone();
let task = tokio::task::spawn_blocking(move || {
let con = task_con.lock().map_err(|e| e.to_string())?;
duckdb_execute_for_database(&con, &attached_names, database.as_deref(), &sql, max_rows)
});
let result =
wait_for_duckdb_task_with_interrupt_outcome(cancel_token, query_timeout, interrupt_handle, task).await;
match result {
DuckDbTaskWait::Finished(result) => {
if matches!(result.as_ref(), Err(err) if err == QUERY_CANCELED || is_dbx_query_timeout_error(&err.to_lowercase()))
{
con.mark_draining();
state.spawn_duckdb_pool_cleanup(pool_key.to_string(), con);
}
result
}
DuckDbTaskWait::Draining { error, task } => {
con.mark_draining();
state.spawn_duckdb_draining_cleanup(pool_key.to_string(), con, task);
Err(error)
}
}
}
#[cfg(feature = "duckdb-bundled")]
#[cfg(feature = "duckdb-sidecar")]
PoolKind::DuckDbWorker(client) => {
let client = client.clone();
if let Some(ref execution_id) = options.execution_id {
@@ -1445,11 +1001,7 @@ pub async fn do_execute(
drop(connections);
client.execute(database, sql, max_rows, cancel_token, query_timeout).await
}
#[cfg(not(feature = "duckdb-bundled"))]
PoolKind::DuckDb(_) => {
return Err("DuckDB support is not compiled in this build".to_string());
}
#[cfg(not(feature = "duckdb-bundled"))]
#[cfg(not(feature = "duckdb-sidecar"))]
PoolKind::DuckDbWorker(_) => {
return Err("DuckDB worker support is not compiled in this build".to_string());
}
@@ -1719,32 +1271,6 @@ pub async fn do_execute(
}
result
}
#[cfg(feature = "duckdb-bundled")]
PoolKind::ExternalTabular(ext_pool) => {
if !starts_with_duckdb_result_sql_keyword(sql) {
return Err("External data sources are read-only. Only SELECT queries are supported.".to_string());
}
let con = ext_pool.cache.clone();
let interrupt_handle = con.lock().map_err(|e| e.to_string())?.interrupt_handle();
if let Some(ref execution_id) = options.execution_id {
let cancel_interrupt_handle = interrupt_handle.clone();
state.running_queries.register_interrupt(execution_id, move || {
cancel_interrupt_handle.interrupt();
});
}
let sql = sql.to_string();
let max_rows = options.max_rows;
drop(connections);
let task = tokio::task::spawn_blocking(move || {
let con = con.lock().map_err(|e| e.to_string())?;
duckdb_execute_with_max_rows(&con, &sql, max_rows)
});
wait_for_duckdb_task_with_interrupt(cancel_token, query_timeout, interrupt_handle, task).await
}
#[cfg(not(feature = "duckdb-bundled"))]
PoolKind::ExternalTabular(_) => {
Err("External data sources require DuckDB support. Rebuild with default features.".to_string())
}
PoolKind::ExternalDriver { config, session, .. } => {
let config = config.clone();
let session = session.clone();
@@ -2769,7 +2295,6 @@ fn schema_diff_failure_outcome(
}
fn pool_kind_has_transactional_path(pool: &PoolKind) -> bool {
// DuckDb variants always exist (unit stubs when duckdb-bundled is off).
match pool {
PoolKind::Postgres(_)
| PoolKind::Mysql(_, _)
@@ -2783,14 +2308,12 @@ fn pool_kind_has_transactional_path(pool: &PoolKind) -> bool {
PoolKind::MessageQueue
| PoolKind::Nacos
| PoolKind::HBase(_)
| PoolKind::DuckDb(_)
| PoolKind::DuckDbWorker(_)
| PoolKind::Redis(_)
| PoolKind::MongoDb(_)
| PoolKind::Elasticsearch(_)
| PoolKind::VectorDb(_)
| PoolKind::InfluxDb(_)
| PoolKind::ExternalTabular(_)
| PoolKind::ExternalDriver { .. } => false,
}
}
@@ -2980,25 +2503,12 @@ pub async fn execute_statements_in_transaction_on_pool(
| PoolKind::SqlServer(_)
| PoolKind::Agent(_) => TxPath::Explicit,
PoolKind::MessageQueue | PoolKind::Nacos | PoolKind::HBase(_) => TxPath::None,
#[cfg(feature = "duckdb-bundled")]
PoolKind::DuckDb(_)
| PoolKind::DuckDbWorker(_)
PoolKind::DuckDbWorker(_)
| PoolKind::Redis(_)
| PoolKind::MongoDb(_)
| PoolKind::Elasticsearch(_)
| PoolKind::VectorDb(_)
| PoolKind::InfluxDb(_)
| PoolKind::ExternalTabular(_)
| PoolKind::ExternalDriver { .. } => TxPath::None,
#[cfg(not(feature = "duckdb-bundled"))]
PoolKind::DuckDb(_)
| PoolKind::DuckDbWorker(_)
| PoolKind::Redis(_)
| PoolKind::MongoDb(_)
| PoolKind::Elasticsearch(_)
| PoolKind::VectorDb(_)
| PoolKind::InfluxDb(_)
| PoolKind::ExternalTabular(_)
| PoolKind::ExternalDriver { .. } => TxPath::None,
})
};
@@ -4744,192 +4254,6 @@ mod tests {
assert_eq!(result.unwrap_err(), QUERY_CANCELED);
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn duckdb_timeout_interrupts_running_task_without_waiting_for_it_to_finish() {
let con = std::sync::Arc::new(crate::db::duckdb_driver::DuckDbConnection::new(
duckdb::Connection::open_in_memory().unwrap(),
));
let interrupt_handle = con.interrupt_handle();
let running_con = con.clone();
let task = tokio::task::spawn_blocking(move || {
let con = running_con.lock().map_err(|e| e.to_string())?;
duckdb_execute_with_max_rows(&con, "SELECT sum(sin(i::DOUBLE)) FROM range(10000000000) tbl(i)", None)
});
let started = std::time::Instant::now();
let result =
wait_for_duckdb_task_with_interrupt(None, Some(Duration::from_millis(10)), interrupt_handle, task).await;
assert_eq!(result.unwrap_err(), timeout_error());
assert!(started.elapsed() < Duration::from_secs(5));
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn duckdb_cancel_keeps_pool_draining_until_references_drop() {
let dir = std::env::temp_dir().join(format!("dbx-query-duckdb-cancel-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let pool_key = "duckdb-1";
let con = std::sync::Arc::new(crate::db::duckdb_driver::DuckDbConnection::new(
duckdb::Connection::open_in_memory().unwrap(),
));
let extra_reference = con.clone();
state.connections.write().await.insert(pool_key.to_string(), PoolKind::DuckDb(con));
state.configs.write().await.insert(pool_key.to_string(), test_connection_config(DatabaseType::DuckDb));
let token = CancellationToken::new();
let cancel_token = token.clone();
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(10)).await;
cancel_token.cancel();
});
let result = do_execute(
&state,
pool_key,
db::mysql::MySqlQueryDialect::default(),
Some("main"),
"SELECT sum(sin(i::DOUBLE)) FROM range(10000000000) tbl(i)",
None,
Some(token),
QueryExecutionOptions::default(),
)
.await;
assert_eq!(result.unwrap_err(), QUERY_CANCELED);
let still_present = {
let conns = state.connections.read().await;
matches!(conns.get(pool_key), Some(PoolKind::DuckDb(current)) if current.is_draining())
};
assert!(still_present);
drop(extra_reference);
// The draining-cleanup task (spawn_duckdb_draining_cleanup) cannot drop
// the pool until the cancelled query's blocking DuckDB task has fully
// unwound — the connection must stay alive while the query still holds
// it. That unwind is near-instant on an idle runner but can exceed the
// original 5s window under heavy CI load, flaking this test, so allow
// ample headroom.
timeout(Duration::from_secs(30), async {
loop {
if !state.connections.read().await.contains_key(pool_key) {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("draining DuckDB pool should be removed after references drop");
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn duckdb_draining_pool_rejects_follow_up_query() {
let dir = std::env::temp_dir().join(format!("dbx-query-duckdb-draining-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let pool_key = "duckdb-1";
let con = std::sync::Arc::new(crate::db::duckdb_driver::DuckDbConnection::new(
duckdb::Connection::open_in_memory().unwrap(),
));
con.mark_draining();
state.connections.write().await.insert(pool_key.to_string(), PoolKind::DuckDb(con));
state.configs.write().await.insert(pool_key.to_string(), test_connection_config(DatabaseType::DuckDb));
let result = do_execute(
&state,
pool_key,
db::mysql::MySqlQueryDialect::default(),
Some("main"),
"SELECT 1",
None,
None,
QueryExecutionOptions::default(),
)
.await;
assert_eq!(result.unwrap_err(), duckdb_draining_error());
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn duckdb_draining_cleanup_removes_pool_after_task_finishes() {
let dir = std::env::temp_dir().join(format!("dbx-query-duckdb-cleanup-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let pool_key = "duckdb-1";
let con = std::sync::Arc::new(crate::db::duckdb_driver::DuckDbConnection::new(
duckdb::Connection::open_in_memory().unwrap(),
));
con.mark_draining();
state.connections.write().await.insert(pool_key.to_string(), PoolKind::DuckDb(con.clone()));
let task_con = con.clone();
let task = tokio::task::spawn_blocking(move || {
let _locked = task_con.lock().map_err(|e| e.to_string())?;
std::thread::sleep(Duration::from_millis(100));
Ok(empty_query_result(0))
});
state.spawn_duckdb_draining_cleanup(pool_key.to_string(), con.clone(), task);
assert!(state.connections.read().await.contains_key(pool_key));
drop(con);
timeout(Duration::from_secs(5), async {
loop {
if !state.connections.read().await.contains_key(pool_key) {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("draining cleanup should remove the DuckDB pool");
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn duckdb_cleanup_keeps_draining_pool_while_extra_reference_exists() {
let dir = std::env::temp_dir().join(format!("dbx-query-duckdb-cleanup-ref-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let pool_key = "duckdb-1";
let con = std::sync::Arc::new(crate::db::duckdb_driver::DuckDbConnection::new(
duckdb::Connection::open_in_memory().unwrap(),
));
con.mark_draining();
state.connections.write().await.insert(pool_key.to_string(), PoolKind::DuckDb(con.clone()));
let extra_reference = con.clone();
let task = tokio::task::spawn_blocking(|| Ok(empty_query_result(0)));
state.spawn_duckdb_draining_cleanup(pool_key.to_string(), con.clone(), task);
tokio::time::sleep(Duration::from_millis(100)).await;
let still_present = {
let conns = state.connections.read().await;
matches!(conns.get(pool_key), Some(PoolKind::DuckDb(current)) if current.is_draining())
};
assert!(still_present);
drop(extra_reference);
drop(con);
timeout(Duration::from_secs(5), async {
loop {
if !state.connections.read().await.contains_key(pool_key) {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("draining cleanup should remove the DuckDB pool after extra refs drop");
}
#[test]
fn db_operation_budget_from_config() {
let budget = DbOperationBudget::from_config(10, Some(30));
@@ -5162,150 +4486,6 @@ mod tests {
assert_eq!(pool_error_action(Some(DatabaseType::Dameng), dameng_err), PoolErrorAction::ReconnectAndRetry);
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_preserves_double_precision() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(
&con,
"SELECT 12.34567::DOUBLE AS sample, 0.5::DOUBLE AS half, 99.99::DOUBLE AS price, 1.0::DOUBLE AS one",
)
.expect("execute double query");
assert_eq!(result.columns, vec!["sample", "half", "price", "one"]);
let row = &result.rows[0];
assert_eq!(row[0], serde_json::json!(12.34567));
assert_eq!(row[1], serde_json::json!(0.5));
assert_eq!(row[2], serde_json::json!(99.99));
assert_eq!(row[3], serde_json::json!(1.0));
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_create_insert_select_double() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
con.execute_batch("CREATE TABLE tmp1 (tmp_double DOUBLE)").expect("create table");
con.execute_batch("INSERT INTO tmp1 VALUES (45.678), (12.345), (99.999)").expect("insert");
let result = duckdb_execute(&con, "SELECT tmp_double FROM tmp1 ORDER BY tmp_double").expect("select doubles");
assert_eq!(result.rows.len(), 3);
assert_eq!(result.rows[0][0], serde_json::json!(12.345));
assert_eq!(result.rows[1][0], serde_json::json!(45.678));
assert_eq!(result.rows[2][0], serde_json::json!(99.999));
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_returns_rows_for_from_first_query() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
con.execute_batch("CREATE TABLE users (id INTEGER, name VARCHAR)").expect("create table");
con.execute_batch("INSERT INTO users VALUES (2, 'Grace'), (1, 'Ada')").expect("insert");
let result = duckdb_execute(&con, "FROM users ORDER BY id").expect("execute from-first query");
assert_eq!(result.columns, vec!["id", "name"]);
assert_eq!(result.rows.len(), 2);
assert_eq!(result.rows[0], vec![serde_json::json!(1), serde_json::json!("Ada")]);
assert_eq!(result.rows[1], vec![serde_json::json!(2), serde_json::json!("Grace")]);
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_returns_rows_for_summarize_query() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
con.execute_batch("CREATE TABLE metrics (value INTEGER)").expect("create table");
con.execute_batch("INSERT INTO metrics VALUES (1), (2), (NULL)").expect("insert");
let result = duckdb_execute(&con, "SUMMARIZE metrics").expect("execute summarize query");
assert!(!result.columns.is_empty());
assert!(!result.rows.is_empty());
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_handles_various_types() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(
&con,
"SELECT 42 AS int_val, true AS bool_val, 'hello' AS text_val, 3.14::FLOAT AS float_val, 123456789012345::BIGINT AS big_val",
)
.expect("execute mixed types query");
let row = &result.rows[0];
assert_eq!(row[0], serde_json::json!(42));
assert_eq!(row[1], serde_json::json!(true));
assert_eq!(row[2], serde_json::Value::String("hello".to_string()));
assert!(row[3].is_number());
assert_eq!(row[4], serde_json::json!(123456789012345_i64));
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_returns_list_values_as_json_arrays() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(&con, "SELECT ['a','b','c','d'];").expect("execute list query");
assert_eq!(result.rows, vec![vec![serde_json::json!(["a", "b", "c", "d"])]]);
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_preserves_nulls_inside_list_values() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(&con, "SELECT [1, NULL, 3] AS items;").expect("execute nullable list query");
assert_eq!(result.columns, vec!["items"]);
assert_eq!(result.rows, vec![vec![serde_json::json!([1, null, 3])]]);
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_returns_nested_complex_values_as_json() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(
&con,
"SELECT {'name': 'Ada', 'scores': [10, 20]} AS profile, MAP(['x', 'y'], [1, 2]) AS lookup, [1, 2, 3]::INTEGER[3] AS fixed_items",
)
.expect("execute complex values query");
assert_eq!(result.columns, vec!["profile", "lookup", "fixed_items"]);
assert_eq!(
result.rows,
vec![vec![
serde_json::json!({ "name": "Ada", "scores": [10, 20] }),
serde_json::json!([
{ "key": "x", "value": 1 },
{ "key": "y", "value": 2 },
]),
serde_json::json!([1, 2, 3]),
]]
);
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_formats_temporal_values_by_column_type() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(
&con,
"SELECT DATE '2026-05-14' AS d, TIME '16:58:15' AS t, TIMESTAMP '2026-05-14 16:58:15.0' AS ts, NULL::TIMESTAMP AS nts",
)
.expect("execute temporal query");
assert_eq!(result.columns, vec!["d", "t", "ts", "nts"]);
assert_eq!(
result.rows,
vec![vec![
serde_json::Value::String("2026-05-14".to_string()),
serde_json::Value::String("16:58:15".to_string()),
serde_json::Value::String("2026-05-14 16:58:15".to_string()),
serde_json::Value::Null,
]]
);
}
#[test]
fn external_driver_query_params_include_database_and_schema_context() {
let config = ConnectionConfig {
File diff suppressed because it is too large. Load diff
@@ -1,203 +0,0 @@
use crate::connection::AppState;
use crate::db;
pub fn duckdb_query_tables(con: &duckdb::Connection) -> Result<Vec<db::TableInfo>, String> {
duckdb_query_tables_in_database(con, "main", "main")
}
pub fn duckdb_query_tables_in_database(
con: &duckdb::Connection,
database: &str,
schema: &str,
) -> Result<Vec<db::TableInfo>, String> {
duckdb_query_tables_in_database_with_attached(con, database, schema, &[])
}
pub fn duckdb_query_tables_in_database_with_attached(
con: &duckdb::Connection,
database: &str,
schema: &str,
attached_names: &[String],
) -> Result<Vec<db::TableInfo>, String> {
let database = duckdb_catalog_name(con, database, attached_names)?;
let mut stmt = con
.prepare(
"SELECT table_name, table_type FROM information_schema.tables WHERE table_catalog = ? AND table_schema = ? ORDER BY table_name",
)
.map_err(|e| e.to_string())?;
let rows = stmt
.query_map((database.as_str(), schema), |row| {
Ok(db::TableInfo {
name: row.get::<_, String>(0)?,
table_type: row.get::<_, String>(1)?,
comment: None,
parent_schema: None,
parent_name: None,
})
})
.map_err(|e| e.to_string())?;
Ok(rows.filter_map(|r| r.ok()).collect())
}
pub fn duckdb_attach_database(con: &duckdb::Connection, name: &str, path: &str) -> Result<(), String> {
let name = name.trim();
let path = path.trim();
if name.is_empty() || path.is_empty() {
return Err("DuckDB attached database name and path are required".to_string());
}
let sql = format!("ATTACH {} AS {}", duckdb_quote_string(path), duckdb_quote_ident(name));
con.execute_batch(&sql).map_err(|e| e.to_string())
}
pub fn duckdb_list_databases(con: &duckdb::Connection) -> Result<Vec<db::DatabaseInfo>, String> {
duckdb_list_databases_with_attached(con, &[])
}
pub fn duckdb_list_databases_with_attached(
con: &duckdb::Connection,
attached_names: &[String],
) -> Result<Vec<db::DatabaseInfo>, String> {
let primary = duckdb_primary_catalog(con, attached_names)?;
let mut stmt = con.prepare("SHOW DATABASES").map_err(|e| e.to_string())?;
let rows = stmt
.query_map([], |row| {
let name = row.get::<_, String>(0)?;
Ok(db::DatabaseInfo { name: if name == primary { "main".to_string() } else { name } })
})
.map_err(|e| e.to_string())?;
Ok(rows.filter_map(|row| row.ok()).collect())
}
pub fn duckdb_list_schemas(con: &duckdb::Connection, database: &str) -> Result<Vec<String>, String> {
duckdb_list_schemas_with_attached(con, database, &[])
}
pub fn duckdb_list_schemas_with_attached(
con: &duckdb::Connection,
database: &str,
attached_names: &[String],
) -> Result<Vec<String>, String> {
let database = duckdb_catalog_name(con, database, attached_names)?;
let mut stmt = con
.prepare(
"SELECT schema_name FROM information_schema.schemata WHERE catalog_name = ? AND schema_name NOT IN ('information_schema', 'pg_catalog') ORDER BY schema_name",
)
.map_err(|e| e.to_string())?;
let rows = stmt.query_map([database.as_str()], |row| row.get::<_, String>(0)).map_err(|e| e.to_string())?;
Ok(rows.filter_map(|r| r.ok()).collect())
}
fn duckdb_catalog_name(con: &duckdb::Connection, database: &str, attached_names: &[String]) -> Result<String, String> {
if database.trim().is_empty() || database == "main" {
return duckdb_primary_catalog(con, attached_names);
}
Ok(database.to_string())
}
pub fn duckdb_primary_catalog(con: &duckdb::Connection, attached_names: &[String]) -> Result<String, String> {
if attached_names.is_empty() {
return duckdb_current_database(con);
}
let attached: std::collections::HashSet<String> = attached_names.iter().map(|name| name.to_lowercase()).collect();
let mut stmt = con.prepare("SHOW DATABASES").map_err(|e| e.to_string())?;
let rows = stmt.query_map([], |row| row.get::<_, String>(0)).map_err(|e| e.to_string())?;
for row in rows {
let name = row.map_err(|e| e.to_string())?;
if !attached.contains(&name.to_lowercase()) {
return Ok(name);
}
}
duckdb_current_database(con)
}
fn duckdb_current_database(con: &duckdb::Connection) -> Result<String, String> {
con.query_row("SELECT current_database()", [], |row| row.get::<_, String>(0)).map_err(|e| e.to_string())
}
fn duckdb_quote_ident(value: &str) -> String {
format!("\"{}\"", value.replace('"', "\"\""))
}
fn duckdb_quote_string(value: &str) -> String {
format!("'{}'", value.replace('\'', "''"))
}
pub fn duckdb_query_columns(con: &duckdb::Connection, table: &str) -> Result<Vec<db::ColumnInfo>, String> {
duckdb_query_columns_in_database(con, "main", "main", table)
}
pub fn duckdb_query_columns_in_database(
con: &duckdb::Connection,
database: &str,
schema: &str,
table: &str,
) -> Result<Vec<db::ColumnInfo>, String> {
duckdb_query_columns_in_database_with_attached(con, database, schema, table, &[])
}
pub fn duckdb_query_columns_in_database_with_attached(
con: &duckdb::Connection,
database: &str,
schema: &str,
table: &str,
attached_names: &[String],
) -> Result<Vec<db::ColumnInfo>, String> {
let database = duckdb_catalog_name(con, database, attached_names)?;
let mut pk_stmt = con
.prepare(
"SELECT kcu.column_name
FROM information_schema.table_constraints tc
JOIN information_schema.key_column_usage kcu
ON tc.constraint_name = kcu.constraint_name
AND tc.table_schema = kcu.table_schema
AND tc.table_name = kcu.table_name
WHERE tc.constraint_type = 'PRIMARY KEY'
AND tc.table_catalog = ?
AND tc.table_schema = ?
AND tc.table_name = ?
ORDER BY kcu.ordinal_position",
)
.map_err(|e| e.to_string())?;
let pk_rows = pk_stmt
.query_map((database.as_str(), schema, table), |row| row.get::<_, String>(0))
.map_err(|e| e.to_string())?;
let primary_keys: std::collections::HashSet<String> = pk_rows.filter_map(|r| r.ok()).collect();
let mut stmt = con
.prepare(
"SELECT column_name, data_type, is_nullable, column_default
FROM information_schema.columns
WHERE table_catalog = ? AND table_schema = ? AND table_name = ?
ORDER BY ordinal_position",
)
.map_err(|e| e.to_string())?;
let rows = stmt
.query_map((database.as_str(), schema, table), |row| {
let name = row.get::<_, String>(0)?;
Ok(db::ColumnInfo {
is_primary_key: primary_keys.contains(&name),
name,
data_type: row.get::<_, String>(1)?,
is_nullable: row.get::<_, String>(2).unwrap_or_default() == "YES",
column_default: row.get::<_, Option<String>>(3)?,
extra: None,
comment: None,
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
..Default::default()
})
})
.map_err(|e| e.to_string())?;
Ok(rows.filter_map(|r| r.ok()).collect())
}
pub(super) async fn duckdb_attached_database_names(state: &AppState, connection_id: &str) -> Vec<String> {
state
.configs
.read()
.await
.get(connection_id)
.map(crate::db::duckdb_sql::config_attached_names)
.unwrap_or_default()
}
@@ -6,7 +6,7 @@ pub(super) enum StructureDialect {
Doris,
Postgres,
Sqlite,
#[cfg(feature = "duckdb-bundled")]
#[cfg(feature = "duckdb-sidecar")]
DuckDb,
SqlServer,
Oracle,
@@ -171,7 +171,7 @@ pub(super) fn capabilities_for(database_type: Option<DatabaseType>) -> TableStru
index_filter: true,
..base
},
#[cfg(feature = "duckdb-bundled")]
#[cfg(feature = "duckdb-sidecar")]
Some(DatabaseType::DuckDb) => TableStructureCapabilities {
dialect: StructureDialect::DuckDb,
add_column: true,
@@ -305,7 +305,7 @@ pub(super) fn dialect_label(dialect: StructureDialect) -> String {
StructureDialect::Doris => "doris",
StructureDialect::Postgres => "postgres",
StructureDialect::Sqlite => "sqlite",
#[cfg(feature = "duckdb-bundled")]
#[cfg(feature = "duckdb-sidecar")]
StructureDialect::DuckDb => "duckdb",
StructureDialect::SqlServer => "sqlserver",
StructureDialect::Oracle => "oracle",
@@ -326,7 +326,7 @@ pub(super) fn database_type_for_dialect(dialect: StructureDialect) -> Option<Dat
StructureDialect::Doris => Some(DatabaseType::Doris),
StructureDialect::Postgres => Some(DatabaseType::Postgres),
StructureDialect::Sqlite => Some(DatabaseType::Sqlite),
#[cfg(feature = "duckdb-bundled")]
#[cfg(feature = "duckdb-sidecar")]
StructureDialect::DuckDb => Some(DatabaseType::DuckDb),
StructureDialect::SqlServer => Some(DatabaseType::SqlServer),
StructureDialect::Oracle => Some(DatabaseType::Oracle),
@@ -2465,7 +2465,7 @@ fn sqlserver_existing_column_identity_change_warns_without_unchanged_foreign_key
);
}
#[cfg(feature = "duckdb-bundled")]
#[cfg(feature = "duckdb-sidecar")]
#[test]
fn builds_duckdb_create_table_statements() {
let mut name = column("name");
+10 -193
View File
@@ -10,8 +10,6 @@ use crate::models::connection::DatabaseType;
use crate::object_source_sql::{build_executable_object_source_statements, EditableObjectSourceSqlInput};
use crate::query::{agent_execute_query_params, pool_error_action, PoolErrorAction, QueryExecutionOptions};
use crate::sql::split_sql_statements;
#[cfg(feature = "duckdb-bundled")]
use crate::sql::starts_with_executable_sql_keyword;
use crate::sql_dialect::{qualified_transfer_table, quote_transfer_identifier};
static CANCELLED: std::sync::LazyLock<RwLock<HashSet<String>>> =
@@ -3048,89 +3046,12 @@ async fn execute_on_pool_once(
);
client.execute_query(params).await
}
#[cfg(feature = "duckdb-bundled")]
PoolKind::DuckDb(con) => {
let con = con.clone();
#[cfg(feature = "duckdb-sidecar")]
PoolKind::DuckDbWorker(client) => {
let client = client.clone();
let sql = sql.to_string();
drop(connections);
tokio::task::spawn_blocking(move || {
let con = con.lock().map_err(|e| e.to_string())?;
if max_rows.is_some()
&& starts_with_executable_sql_keyword(&sql, &["SELECT", "SHOW", "DESCRIBE", "WITH", "PRAGMA"])
{
return crate::query::duckdb_execute_with_max_rows(&con, &sql, max_rows);
}
let start = std::time::Instant::now();
if starts_with_executable_sql_keyword(&sql, &["SELECT", "SHOW", "DESCRIBE", "WITH", "PRAGMA"]) {
let mut stmt = con.prepare(&sql).map_err(|e| e.to_string())?;
let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
let stmt_ref = rows.as_ref().ok_or("DuckDB statement unavailable")?;
let col_count = stmt_ref.column_count();
let columns: Vec<String> = (0..col_count)
.map(|i| stmt_ref.column_name(i).map(|s| s.to_string()).unwrap_or_else(|_| "?".to_string()))
.collect();
let mut result_rows = Vec::new();
while let Some(row) = rows.next().map_err(|e| e.to_string())? {
let vals: Vec<serde_json::Value> = (0..col_count)
.map(|i| {
row.get::<_, String>(i)
.map(serde_json::Value::String)
.or_else(|_| row.get::<_, i64>(i).map(|v| serde_json::Value::Number(v.into())))
.or_else(|_| {
row.get::<_, f64>(i).map(|v| {
serde_json::Number::from_f64(v)
.map(serde_json::Value::Number)
.unwrap_or(serde_json::Value::Null)
})
})
.or_else(|_| row.get::<_, bool>(i).map(serde_json::Value::Bool))
.unwrap_or(serde_json::Value::Null)
})
.collect();
result_rows.push(vals);
}
Ok(db::QueryResult {
columns,
column_types: Vec::new(),
column_sortables: vec![],
rows: result_rows,
affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(),
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
})
} else {
let affected = con.execute(&sql, []).map_err(|e| e.to_string())?;
Ok(db::QueryResult {
columns: vec![],
column_types: Vec::new(),
column_sortables: vec![],
rows: vec![],
affected_rows: affected as u64,
execution_time_ms: start.elapsed().as_millis(),
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
})
}
})
.await
.map_err(|e| e.to_string())?
}
#[cfg(feature = "duckdb-bundled")]
PoolKind::ExternalTabular(ext_pool) => {
let con = ext_pool.cache.clone();
let sql = sql.to_string();
drop(connections);
tokio::task::spawn_blocking(move || {
let con = con.lock().map_err(|e| e.to_string())?;
crate::query::duckdb_execute_with_max_rows(&con, &sql, max_rows)
})
.await
.map_err(|e| e.to_string())?
client.execute(None, sql, max_rows, None, None).await
}
_ => Err("Unsupported database type for transfer".to_string()),
}
@@ -3161,32 +3082,14 @@ pub async fn get_columns_for_transfer(
) -> Result<Vec<db::ColumnInfo>, String> {
let connections = state.connections.read().await;
#[cfg(feature = "duckdb-bundled")]
if let Some(PoolKind::DuckDb(con)) = connections.get(pool_key) {
let con = con.clone();
drop(connections);
let table = table.to_string();
#[cfg(feature = "duckdb-sidecar")]
if let Some(PoolKind::DuckDbWorker(client)) = connections.get(pool_key) {
let client = client.clone();
let database = database.to_string();
let schema = schema.to_string();
return tokio::task::spawn_blocking(move || {
let con = con.lock().map_err(|e| e.to_string())?;
crate::schema::duckdb_query_columns_in_database(&con, "main", &schema, &table)
})
.await
.map_err(|e| e.to_string())?;
}
#[cfg(feature = "duckdb-bundled")]
if let Some(PoolKind::ExternalTabular(ext_pool)) = connections.get(pool_key) {
let con = ext_pool.cache.clone();
drop(connections);
let table = table.to_string();
let schema = schema.to_string();
return tokio::task::spawn_blocking(move || {
let con = con.lock().map_err(|e| e.to_string())?;
crate::schema::duckdb_query_columns_in_database(&con, "main", &schema, &table)
})
.await
.map_err(|e| e.to_string())?;
drop(connections);
return client.list_columns(database, schema, table).await;
}
if let Some(PoolKind::ClickHouse(client)) = connections.get(pool_key) {
@@ -5029,73 +4932,7 @@ where
#[cfg(test)]
mod tests {
use super::*;
#[cfg(feature = "duckdb-bundled")]
use crate::connection::{AppState, PoolKind};
#[cfg(feature = "duckdb-bundled")]
use crate::models::connection::default_redis_key_separator;
#[cfg(feature = "duckdb-bundled")]
use crate::storage::Storage;
use serde_json::json;
#[cfg(feature = "duckdb-bundled")]
use std::sync::Arc;
#[cfg(feature = "duckdb-bundled")]
fn duckdb_test_config(id: &str) -> crate::models::connection::ConnectionConfig {
crate::models::connection::ConnectionConfig {
id: id.to_string(),
name: id.to_string(),
note: String::new(),
db_type: DatabaseType::DuckDb,
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: ":memory:".to_string(),
port: 0,
username: String::new(),
password: String::new(),
database: None,
visible_databases: None,
visible_schemas: None,
show_system_schemas: false,
attached_databases: Vec::new(),
init_script: None,
color: None,
transport_layers: Vec::new(),
connect_timeout_secs: 5,
query_timeout_secs: 30,
idle_timeout_secs: 60,
keepalive_interval_secs: 0,
ssl: false,
ca_cert_path: String::new(),
client_cert_path: String::new(),
client_key_path: String::new(),
sysdba: false,
oracle_connection_type: None,
connection_string: None,
redis_connection_mode: None,
redis_sentinel_master: String::new(),
redis_sentinel_nodes: String::new(),
redis_sentinel_username: String::new(),
redis_sentinel_password: String::new(),
redis_sentinel_tls: false,
redis_cluster_nodes: String::new(),
redis_key_separator: default_redis_key_separator(),
redis_scan_page_size: None,
redis_database_aliases: Default::default(),
etcd_endpoints: String::new(),
gbase_server: String::new(),
informix_server: String::new(),
external_config: None,
jdbc_driver_class: None,
jdbc_driver_paths: Vec::new(),
one_time: false,
read_only: false,
is_production: false,
production_databases: vec![],
database_info: None,
}
}
fn test_column(name: &str, data_type: &str) -> db::ColumnInfo {
db::ColumnInfo {
@@ -6863,26 +6700,6 @@ SELECT 1 FROM dual"#
assert!(statements[0].contains("ON DUPLICATE KEY UPDATE"));
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test]
async fn duckdb_transfer_columns_use_requested_schema() {
let dir = std::env::temp_dir().join(format!("dbx-transfer-test-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let con = duckdb::Connection::open_in_memory().unwrap();
con.execute_batch("CREATE SCHEMA analytics; CREATE TABLE analytics.items(id INTEGER);").unwrap();
let state = AppState::new(storage);
let con = Arc::new(crate::db::duckdb_driver::DuckDbConnection::new(con));
state.connections.write().await.insert("duckdb-1".to_string(), PoolKind::DuckDb(con));
state.configs.write().await.insert("duckdb-1".to_string(), duckdb_test_config("duckdb-1"));
let columns =
get_columns_for_transfer(&state, "duckdb-1", "duckdb-1", "main", "analytics", "items").await.unwrap();
assert_eq!(columns.iter().map(|c| c.name.as_str()).collect::<Vec<_>>(), vec!["id"]);
}
#[test]
fn database_from_pool_key_handles_session_scoped_keys() {
assert_eq!(database_from_pool_key("conn:analytics"), Some("analytics"));