feat(duckdb): move support to standalone driver
This commit is contained in:
76 files changed
+5400
-8258
No files matched your search
@@ -40,12 +40,14 @@ const H2_PROFILES: &[AgentDriverProfile] =
|
||||
&[AgentDriverProfile { profile: "h2-legacy", key: "h2-legacy", label: "H2 2.1 Legacy", store_visible: true }];
|
||||
|
||||
const EXTRA_AGENT_LABELS: &[(&str, &str)] = &[
|
||||
("duckdb", "DuckDB"),
|
||||
("kafka", "Apache Kafka"),
|
||||
("rocketmq", "Apache RocketMQ"),
|
||||
("rabbitmq", "RabbitMQ"),
|
||||
("sqlserver-legacy", "SQL Server legacy compatibility component"),
|
||||
];
|
||||
const EXTRA_DRIVER_STORE_ENTRIES: &[(&str, &str)] = &[
|
||||
("duckdb", "DuckDB"),
|
||||
("kafka", "Apache Kafka"),
|
||||
("rocketmq", "Apache RocketMQ"),
|
||||
("rabbitmq", "RabbitMQ"),
|
||||
@@ -388,4 +390,11 @@ mod tests {
|
||||
assert_eq!(agent_key(&DatabaseType::H2, Some("h2-legacy")), Some("h2-legacy"));
|
||||
assert!(driver_store_entries().any(|(key, label)| key == "h2-legacy" && label == "H2 2.1 Legacy"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duckdb_is_available_in_driver_store_without_using_agent_runtime() {
|
||||
assert!(driver_store_entries().any(|(key, label)| key == "duckdb" && label == "DuckDB"));
|
||||
assert_eq!(label_for_key("duckdb"), Some("DuckDB"));
|
||||
assert!(!is_agent_type(&DatabaseType::DuckDb));
|
||||
}
|
||||
}
|
||||
@@ -345,6 +345,14 @@ pub struct DriverInfo {
|
||||
pub struct ArtifactInfo {
|
||||
pub url: String,
|
||||
pub size: u64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub format: Option<ArtifactFormat>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ArtifactFormat {
|
||||
TarZstd,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
|
||||
@@ -8,7 +8,7 @@ use futures::{stream, StreamExt};
|
||||
|
||||
use crate::agent_catalog;
|
||||
use crate::agent_manager::{
|
||||
AgentDriverInfo, AgentManager, AgentRegistry, InstalledDriver, JavaRuntimeMode, DEFAULT_JRE_KEY,
|
||||
AgentDriverInfo, AgentManager, AgentRegistry, ArtifactFormat, InstalledDriver, JavaRuntimeMode, DEFAULT_JRE_KEY,
|
||||
};
|
||||
use crate::DownloadSource;
|
||||
|
||||
@@ -577,7 +577,7 @@ pub async fn reinstall_agent_jre_from(
|
||||
.platforms
|
||||
.get(platform)
|
||||
.ok_or_else(|| format!("No JRE {jre_key} available for platform: {platform}"))?;
|
||||
let jre_archive = am.base_dir().join("jre-download.tar.gz");
|
||||
let jre_archive = jre_archive_download_path(am, jre_key, platform_jre.format);
|
||||
download_with_progress(
|
||||
am,
|
||||
&progress,
|
||||
@@ -600,7 +600,7 @@ pub async fn reinstall_agent_jre_from(
|
||||
am.stop_daemons().await;
|
||||
let stash = replace_old_jre_dir(&jre_dir)?;
|
||||
persist_pending_jre_cleanup(am, stash.as_ref()).await?;
|
||||
extract_tar_gz(&jre_archive, &jre_dir)?;
|
||||
extract_jre_archive(&jre_archive, &jre_dir, platform_jre.format)?;
|
||||
std::fs::remove_file(&jre_archive).ok();
|
||||
am.mutate_state(|state| state.jre_versions.insert(jre_key.to_string(), jre_info.version.clone()))?;
|
||||
cleanup_jre_download_cache_after_success(am, jre_key);
|
||||
@@ -627,6 +627,41 @@ pub async fn import_agents_from_zip(
|
||||
.await
|
||||
}
|
||||
|
||||
pub fn inspect_offline_package(package_path: &Path) -> Result<OfflineImportPlan, String> {
|
||||
if is_tar_zstd_package(package_path) {
|
||||
inspect_tar_zstd_driver_package(package_path)
|
||||
} else {
|
||||
inspect_offline_zip(package_path)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn import_agents_from_package(
|
||||
am: &AgentManager,
|
||||
package_path: &Path,
|
||||
progress: impl Fn(AgentProgressEvent),
|
||||
) -> Result<OfflineImportResult, String> {
|
||||
if is_tar_zstd_package(package_path) {
|
||||
import_tar_zstd_driver_package(am, package_path, |event| {
|
||||
progress(AgentProgressEvent {
|
||||
operation_id: None,
|
||||
step: event.step,
|
||||
downloaded: Some(event.current as u64),
|
||||
total: Some(event.total as u64),
|
||||
db_type: event.db_type,
|
||||
current: Some(event.current),
|
||||
total_drivers: Some(event.total),
|
||||
});
|
||||
})
|
||||
.await
|
||||
} else {
|
||||
import_agents_from_zip(am, package_path, progress).await
|
||||
}
|
||||
}
|
||||
|
||||
fn is_tar_zstd_package(path: &Path) -> bool {
|
||||
path.file_name().and_then(|name| name.to_str()).is_some_and(|name| name.to_ascii_lowercase().ends_with(".tar.zst"))
|
||||
}
|
||||
|
||||
async fn install_agent_driver_with_batch(
|
||||
am: &AgentManager,
|
||||
db_type: &str,
|
||||
@@ -734,7 +769,7 @@ async fn ensure_jre_from_registry(
|
||||
.platforms
|
||||
.get(platform)
|
||||
.ok_or_else(|| format!("No JRE {jre_key} available for platform: {platform}"))?;
|
||||
let jre_archive = am.base_dir().join(format!("jre-{}-download.tar.gz", jre_key));
|
||||
let jre_archive = jre_archive_download_path(am, jre_key, platform_jre.format);
|
||||
progress(AgentProgressEvent::transfer("jre", 0, platform_jre.size).with_batch(
|
||||
Some(db_type),
|
||||
current,
|
||||
@@ -764,11 +799,11 @@ async fn ensure_jre_from_registry(
|
||||
let stash = replace_old_jre_dir(&jre_dir)?;
|
||||
|
||||
// Persist the stash path *before* extraction so that a crash during
|
||||
// extract_tar_gz (or a process kill) doesn't leave the renamed-stash
|
||||
// archive extraction (or a process kill) doesn't leave the renamed-stash
|
||||
// directory as an orphan that never gets cleaned up.
|
||||
persist_pending_jre_cleanup(am, stash.as_ref()).await?;
|
||||
|
||||
extract_tar_gz(&jre_archive, &jre_dir)?;
|
||||
extract_jre_archive(&jre_archive, &jre_dir, platform_jre.format)?;
|
||||
std::fs::remove_file(&jre_archive).ok();
|
||||
cleanup_jre_download_cache_after_success(am, jre_key);
|
||||
|
||||
@@ -893,6 +928,8 @@ async fn install_agent_driver_from_registry(
|
||||
if let Some(parent) = target_path.parent() {
|
||||
std::fs::create_dir_all(parent).map_err(|err| format!("Failed to create driver directory: {err}"))?;
|
||||
}
|
||||
let artifact_kind = if is_native_artifact { DriverArtifactKind::Native } else { DriverArtifactKind::Jar };
|
||||
let download_path = driver_artifact_download_path(&target_path, artifact.format);
|
||||
progress(AgentProgressEvent::transfer("driver", 0, artifact.size).with_batch(
|
||||
Some(db_type),
|
||||
current,
|
||||
@@ -905,7 +942,7 @@ async fn install_agent_driver_from_registry(
|
||||
source,
|
||||
&artifact.url,
|
||||
&r2_path_with_cache_buster(&github_url_to_r2_path(&artifact.url, "driver"), &driver.version),
|
||||
&target_path,
|
||||
&download_path,
|
||||
artifact.size,
|
||||
Some(CacheIdentity::Driver { db_type, version: &driver.version }),
|
||||
Some(db_type),
|
||||
@@ -913,16 +950,24 @@ async fn install_agent_driver_from_registry(
|
||||
total_drivers,
|
||||
)
|
||||
.await?;
|
||||
install_downloaded_driver_artifact(
|
||||
&download_path,
|
||||
&target_path,
|
||||
artifact.format,
|
||||
artifact_kind,
|
||||
db_type,
|
||||
&driver.version,
|
||||
)?;
|
||||
// Some drivers publish both a native agent and a legacy JAR fallback. Only
|
||||
// validate the artifact type that was actually installed.
|
||||
if !is_native_artifact && !am.is_driver_jar_valid(db_type) {
|
||||
std::fs::remove_file(&target_path).ok();
|
||||
return Err(format!("Downloaded driver jar is invalid or corrupt: {}", target_path.display()));
|
||||
}
|
||||
if is_native_artifact {
|
||||
mark_executable(&target_path)?;
|
||||
std::fs::remove_file(am.driver_jar_path(db_type)).ok();
|
||||
} else {
|
||||
if !am.is_driver_jar_valid(db_type) {
|
||||
std::fs::remove_file(&target_path).ok();
|
||||
return Err(format!("Downloaded driver jar is invalid or corrupt: {}", target_path.display()));
|
||||
}
|
||||
std::fs::remove_file(am.driver_native_path(db_type)).ok();
|
||||
}
|
||||
|
||||
@@ -947,6 +992,169 @@ async fn install_agent_driver_from_registry(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn driver_artifact_download_path(target_path: &Path, format: Option<ArtifactFormat>) -> PathBuf {
|
||||
match format {
|
||||
Some(ArtifactFormat::TarZstd) => {
|
||||
let file_name = target_path.file_name().and_then(|name| name.to_str()).unwrap_or("agent");
|
||||
target_path.with_file_name(format!(".{file_name}.tar.zst"))
|
||||
}
|
||||
None => target_path.to_path_buf(),
|
||||
}
|
||||
}
|
||||
|
||||
fn install_downloaded_driver_artifact(
|
||||
download_path: &Path,
|
||||
target_path: &Path,
|
||||
format: Option<ArtifactFormat>,
|
||||
artifact_kind: DriverArtifactKind,
|
||||
db_type: &str,
|
||||
expected_version: &str,
|
||||
) -> Result<(), String> {
|
||||
let Some(format) = format else {
|
||||
return Ok(());
|
||||
};
|
||||
let result = match format {
|
||||
ArtifactFormat::TarZstd => {
|
||||
install_driver_from_tar_zstd_package(download_path, target_path, artifact_kind, db_type, expected_version)
|
||||
}
|
||||
};
|
||||
if result.is_ok() {
|
||||
std::fs::remove_file(download_path).ok();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn install_driver_from_tar_zstd_package(
|
||||
package_path: &Path,
|
||||
target_path: &Path,
|
||||
expected_kind: DriverArtifactKind,
|
||||
db_type: &str,
|
||||
expected_version: &str,
|
||||
) -> Result<(), String> {
|
||||
let info = tar_zstd_driver_package_info(package_path)?;
|
||||
if info.db_type != db_type {
|
||||
return Err(format!("Driver package contains {}, expected {db_type}", info.db_type));
|
||||
}
|
||||
if info.version != expected_version {
|
||||
return Err(format!(
|
||||
"Driver package version mismatch for {db_type}: expected {expected_version}, got {}",
|
||||
info.version
|
||||
));
|
||||
}
|
||||
if info.kind != expected_kind {
|
||||
return Err(format!(
|
||||
"Driver package artifact type mismatch for {db_type}: expected {}, got {}",
|
||||
expected_kind.label(),
|
||||
info.kind.label()
|
||||
));
|
||||
}
|
||||
let parent = target_path.parent().ok_or_else(|| format!("Invalid driver path: {}", target_path.display()))?;
|
||||
std::fs::create_dir_all(parent).map_err(|error| format!("Failed to create driver directory: {error}"))?;
|
||||
let staging_path = parent.join(format!(".agent-package-{}", uuid::Uuid::new_v4()));
|
||||
let result = extract_tar_zstd_file(package_path, &info.entry_name, &staging_path, info.size).and_then(|_| {
|
||||
match info.kind {
|
||||
DriverArtifactKind::Jar if !is_valid_agent_jar(&staging_path) => {
|
||||
return Err(format!("Packaged driver jar is invalid or corrupt: {}", info.entry_name));
|
||||
}
|
||||
DriverArtifactKind::Native => {
|
||||
validate_native_agent_binary(&staging_path)?;
|
||||
mark_executable(&staging_path)?;
|
||||
}
|
||||
DriverArtifactKind::Jar => {}
|
||||
}
|
||||
replace_download(&staging_path, target_path)
|
||||
});
|
||||
if result.is_err() {
|
||||
std::fs::remove_file(&staging_path).ok();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn read_registry_from_tar_zstd(package_path: &Path) -> Result<AgentRegistry, String> {
|
||||
const MAX_REGISTRY_BYTES: u64 = 1024 * 1024;
|
||||
|
||||
let file = std::fs::File::open(package_path).map_err(|error| format!("Failed to open driver package: {error}"))?;
|
||||
let decoder = zstd::stream::read::Decoder::new(file)
|
||||
.map_err(|error| format!("Failed to open zstd driver package: {error}"))?;
|
||||
let mut archive = tar::Archive::new(decoder);
|
||||
let entries = archive.entries().map_err(|error| format!("Invalid tar.zst driver package: {error}"))?;
|
||||
for entry in entries {
|
||||
let mut entry = entry.map_err(|error| format!("Invalid tar.zst driver package entry: {error}"))?;
|
||||
let path = entry.path().map_err(|error| format!("Invalid driver package path: {error}"))?;
|
||||
let name = safe_archive_entry_name(&path)?;
|
||||
if name != "agent-registry.json" {
|
||||
continue;
|
||||
}
|
||||
if !entry.header().entry_type().is_file() {
|
||||
return Err("Driver package registry is not a regular file".to_string());
|
||||
}
|
||||
if entry.size() > MAX_REGISTRY_BYTES {
|
||||
return Err("Driver package registry is too large".to_string());
|
||||
}
|
||||
let mut json = String::new();
|
||||
entry.read_to_string(&mut json).map_err(|error| format!("Failed to read driver package registry: {error}"))?;
|
||||
return serde_json::from_str(&json).map_err(|error| format!("Invalid driver package registry: {error}"));
|
||||
}
|
||||
Err("agent-registry.json not found in the driver package".to_string())
|
||||
}
|
||||
|
||||
fn extract_tar_zstd_file(
|
||||
package_path: &Path,
|
||||
expected_entry: &str,
|
||||
destination: &Path,
|
||||
expected_size: u64,
|
||||
) -> Result<(), String> {
|
||||
let file = std::fs::File::open(package_path).map_err(|error| format!("Failed to open driver package: {error}"))?;
|
||||
let decoder = zstd::stream::read::Decoder::new(file)
|
||||
.map_err(|error| format!("Failed to open zstd driver package: {error}"))?;
|
||||
let mut archive = tar::Archive::new(decoder);
|
||||
let entries = archive.entries().map_err(|error| format!("Invalid tar.zst driver package: {error}"))?;
|
||||
let mut extracted = false;
|
||||
for entry in entries {
|
||||
let mut entry = entry.map_err(|error| format!("Invalid tar.zst driver package entry: {error}"))?;
|
||||
let path = entry.path().map_err(|error| format!("Invalid driver package path: {error}"))?;
|
||||
let name = safe_archive_entry_name(&path)?;
|
||||
if name == "agent-registry.json" || entry.header().entry_type().is_dir() {
|
||||
continue;
|
||||
}
|
||||
if name != expected_entry {
|
||||
return Err(format!("Unexpected file in driver package: {name}"));
|
||||
}
|
||||
if extracted || !entry.header().entry_type().is_file() {
|
||||
return Err(format!("Invalid driver package entry: {name}"));
|
||||
}
|
||||
let mut output =
|
||||
std::fs::File::create(destination).map_err(|error| format!("Failed to create staged driver: {error}"))?;
|
||||
let copied = std::io::copy(&mut entry, &mut output)
|
||||
.map_err(|error| format!("Failed to extract driver package: {error}"))?;
|
||||
std::io::Write::flush(&mut output).map_err(|error| format!("Failed to flush staged driver: {error}"))?;
|
||||
if expected_size > 0 && copied != expected_size {
|
||||
return Err(format!("Packaged driver size mismatch: expected {expected_size} bytes, got {copied} bytes"));
|
||||
}
|
||||
extracted = true;
|
||||
}
|
||||
if extracted {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!("Driver package entry not found: {expected_entry}"))
|
||||
}
|
||||
}
|
||||
|
||||
fn safe_archive_entry_name(path: &Path) -> Result<String, String> {
|
||||
let mut parts = Vec::new();
|
||||
for component in path.components() {
|
||||
match component {
|
||||
std::path::Component::Normal(value) => parts.push(value.to_string_lossy().into_owned()),
|
||||
std::path::Component::CurDir => {}
|
||||
_ => return Err(format!("Unsafe driver package path: {}", path.display())),
|
||||
}
|
||||
}
|
||||
if parts.is_empty() {
|
||||
return Err("Driver package contains an empty path".to_string());
|
||||
}
|
||||
Ok(parts.join("/"))
|
||||
}
|
||||
|
||||
fn agent_registry_driver<'a>(
|
||||
registry: &'a AgentRegistry,
|
||||
db_type: &str,
|
||||
@@ -1416,10 +1624,181 @@ pub struct OfflineImportPlan {
|
||||
pub includes_jre: bool,
|
||||
}
|
||||
|
||||
type OfflineJreEntry = (String, String);
|
||||
type OfflineJreEntry = (String, String, Option<ArtifactFormat>);
|
||||
type OfflineDriverEntry = (String, String, bool);
|
||||
type OfflineArchiveEntries = (Vec<OfflineJreEntry>, Vec<OfflineDriverEntry>);
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum DriverArtifactKind {
|
||||
Jar,
|
||||
Native,
|
||||
}
|
||||
|
||||
impl DriverArtifactKind {
|
||||
fn label(self) -> &'static str {
|
||||
match self {
|
||||
Self::Jar => "jar",
|
||||
Self::Native => "native",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct TarZstdDriverPackageInfo {
|
||||
db_type: String,
|
||||
version: String,
|
||||
jre: String,
|
||||
kind: DriverArtifactKind,
|
||||
entry_name: String,
|
||||
size: u64,
|
||||
}
|
||||
|
||||
fn inspect_tar_zstd_driver_package(package_path: &Path) -> Result<OfflineImportPlan, String> {
|
||||
let info = tar_zstd_driver_package_info(package_path)?;
|
||||
Ok(OfflineImportPlan { driver_keys: vec![info.db_type], includes_jre: false })
|
||||
}
|
||||
|
||||
async fn import_tar_zstd_driver_package(
|
||||
am: &AgentManager,
|
||||
package_path: &Path,
|
||||
progress: impl Fn(OfflineImportProgress),
|
||||
) -> Result<OfflineImportResult, String> {
|
||||
let _installation_guard = am.installation_operation_lock.write().await;
|
||||
let info = tar_zstd_driver_package_info(package_path)?;
|
||||
let mut result =
|
||||
OfflineImportResult { jre_installed: Vec::new(), drivers_installed: Vec::new(), drivers_skipped: Vec::new() };
|
||||
if let Some(installed) = am.load_state().installed_drivers.get(&info.db_type) {
|
||||
if installed.version != "0.1.0-local"
|
||||
&& installed.version != "local"
|
||||
&& !crate::update::is_newer_version(&info.version, &installed.version)
|
||||
{
|
||||
result.drivers_skipped.push(info.db_type);
|
||||
return Ok(result);
|
||||
}
|
||||
}
|
||||
|
||||
progress(OfflineImportProgress {
|
||||
step: "driver".to_string(),
|
||||
current: 1,
|
||||
total: 1,
|
||||
label: agent_catalog::label_for_key(&info.db_type).unwrap_or(&info.db_type).to_string(),
|
||||
db_type: Some(info.db_type.clone()),
|
||||
});
|
||||
let target_path = match info.kind {
|
||||
DriverArtifactKind::Jar => am.driver_jar_path(&info.db_type),
|
||||
DriverArtifactKind::Native => am.driver_native_path(&info.db_type),
|
||||
};
|
||||
install_driver_from_tar_zstd_package(package_path, &target_path, info.kind, &info.db_type, &info.version)?;
|
||||
match info.kind {
|
||||
DriverArtifactKind::Jar => {
|
||||
std::fs::remove_file(am.driver_native_path(&info.db_type)).ok();
|
||||
}
|
||||
DriverArtifactKind::Native => {
|
||||
std::fs::remove_file(am.driver_jar_path(&info.db_type)).ok();
|
||||
}
|
||||
}
|
||||
am.mutate_state(|state| {
|
||||
state.installed_drivers.insert(
|
||||
info.db_type.clone(),
|
||||
InstalledDriver {
|
||||
version: info.version.clone(),
|
||||
installed_at: chrono::Utc::now().to_rfc3339(),
|
||||
jre: info.jre.clone(),
|
||||
},
|
||||
);
|
||||
})?;
|
||||
am.stop_daemon_by_key(&info.db_type).await;
|
||||
result.drivers_installed.push(info.db_type);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn tar_zstd_driver_package_info(package_path: &Path) -> Result<TarZstdDriverPackageInfo, String> {
|
||||
let registry = read_registry_from_tar_zstd(package_path)?;
|
||||
if registry.drivers.len() != 1 {
|
||||
return Err("A tar.zst driver package must contain exactly one driver".to_string());
|
||||
}
|
||||
let (db_type, driver) = registry.drivers.iter().next().expect("checked one driver");
|
||||
validate_offline_driver_key(db_type)?;
|
||||
let platform = AgentManager::current_platform();
|
||||
let native_artifact = driver.native.get(platform);
|
||||
let jar_artifact = driver.jar.as_ref();
|
||||
let (kind, artifact) = match (native_artifact, jar_artifact) {
|
||||
(Some(_), Some(_)) => {
|
||||
return Err("A tar.zst driver package must contain exactly one driver artifact".to_string());
|
||||
}
|
||||
(Some(artifact), None) => (DriverArtifactKind::Native, artifact),
|
||||
(None, Some(artifact)) => (DriverArtifactKind::Jar, artifact),
|
||||
(None, None) if !driver.native.is_empty() => {
|
||||
return Err(format!("Driver package does not support platform: {platform}"));
|
||||
}
|
||||
(None, None) => return Err("A tar.zst driver package contains no driver artifact".to_string()),
|
||||
};
|
||||
if artifact.format.is_some() {
|
||||
return Err("Nested driver packages are not supported".to_string());
|
||||
}
|
||||
let artifact_filename = artifact
|
||||
.url
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.filter(|name| !name.is_empty())
|
||||
.ok_or_else(|| format!("Invalid packaged driver URL: {}", artifact.url))?;
|
||||
let entry_name = format!("drivers/{artifact_filename}");
|
||||
validate_tar_zstd_package_entries(package_path, &entry_name, artifact.size)?;
|
||||
Ok(TarZstdDriverPackageInfo {
|
||||
db_type: db_type.clone(),
|
||||
version: driver.version.clone(),
|
||||
jre: driver.jre.clone(),
|
||||
kind,
|
||||
entry_name,
|
||||
size: artifact.size,
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_tar_zstd_package_entries(
|
||||
package_path: &Path,
|
||||
expected_entry: &str,
|
||||
expected_size: u64,
|
||||
) -> Result<(), String> {
|
||||
let file = std::fs::File::open(package_path).map_err(|error| format!("Failed to open driver package: {error}"))?;
|
||||
let decoder = zstd::stream::read::Decoder::new(file)
|
||||
.map_err(|error| format!("Failed to open zstd driver package: {error}"))?;
|
||||
let mut archive = tar::Archive::new(decoder);
|
||||
let entries = archive.entries().map_err(|error| format!("Invalid tar.zst driver package: {error}"))?;
|
||||
let mut registry_seen = false;
|
||||
let mut driver_seen = false;
|
||||
for entry in entries {
|
||||
let entry = entry.map_err(|error| format!("Invalid tar.zst driver package entry: {error}"))?;
|
||||
let path = entry.path().map_err(|error| format!("Invalid driver package path: {error}"))?;
|
||||
let name = safe_archive_entry_name(&path)?;
|
||||
if entry.header().entry_type().is_dir() {
|
||||
continue;
|
||||
}
|
||||
if !entry.header().entry_type().is_file() {
|
||||
return Err(format!("Driver package contains a non-regular entry: {name}"));
|
||||
}
|
||||
match name.as_str() {
|
||||
"agent-registry.json" if !registry_seen => registry_seen = true,
|
||||
value if value == expected_entry && !driver_seen => {
|
||||
if expected_size > 0 && entry.size() != expected_size {
|
||||
return Err(format!(
|
||||
"Packaged driver size mismatch: expected {expected_size} bytes, got {} bytes",
|
||||
entry.size()
|
||||
));
|
||||
}
|
||||
driver_seen = true;
|
||||
}
|
||||
_ => return Err(format!("Unexpected file in driver package: {name}")),
|
||||
}
|
||||
}
|
||||
if !registry_seen {
|
||||
return Err("agent-registry.json not found in the driver package".to_string());
|
||||
}
|
||||
if !driver_seen {
|
||||
return Err(format!("Driver package entry not found: {expected_entry}"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn inspect_offline_zip(zip_path: &Path) -> Result<OfflineImportPlan, String> {
|
||||
let file = std::fs::File::open(zip_path).map_err(|e| format!("Failed to open ZIP file: {e}"))?;
|
||||
let mut archive = zip::ZipArchive::new(file).map_err(|e| format!("Invalid ZIP file: {e}"))?;
|
||||
@@ -1461,7 +1840,7 @@ pub async fn import_offline_zip(
|
||||
validate_offline_driver_entries(am, &mut archive, &driver_entries)?;
|
||||
let mut current: u32 = 0;
|
||||
|
||||
for (jre_key, entry_name) in &jre_entries {
|
||||
for (jre_key, entry_name, format) in &jre_entries {
|
||||
current += 1;
|
||||
let jre_version = registry.resolve_jre(jre_key).map(|j| j.version.clone());
|
||||
let existing_version = local_state.jre_versions.get(jre_key);
|
||||
@@ -1478,7 +1857,7 @@ pub async fn import_offline_zip(
|
||||
});
|
||||
|
||||
let mut entry = archive.by_name(entry_name).map_err(|e| format!("Failed to read {entry_name}: {e}"))?;
|
||||
let tmp_archive = am.base_dir().join(format!("jre-offline-{jre_key}.tar.gz"));
|
||||
let tmp_archive = am.base_dir().join(format!("jre-offline-{jre_key}{}", jre_archive_suffix(*format)));
|
||||
{
|
||||
let mut out =
|
||||
std::fs::File::create(&tmp_archive).map_err(|e| format!("Failed to create temp file: {e}"))?;
|
||||
@@ -1487,7 +1866,7 @@ pub async fn import_offline_zip(
|
||||
|
||||
let jre_dir = am.jre_dir(jre_key);
|
||||
let staging_dir = am.base_dir().join(format!(".jre-offline-import-{}", uuid::Uuid::new_v4()));
|
||||
if let Err(error) = extract_tar_gz(&tmp_archive, &staging_dir) {
|
||||
if let Err(error) = extract_jre_archive(&tmp_archive, &staging_dir, *format) {
|
||||
std::fs::remove_dir_all(&staging_dir).ok();
|
||||
std::fs::remove_file(&tmp_archive).ok();
|
||||
return Err(error);
|
||||
@@ -1599,7 +1978,7 @@ fn collect_offline_entries(
|
||||
registry: &AgentRegistry,
|
||||
) -> Result<OfflineArchiveEntries, String> {
|
||||
let platform = AgentManager::current_platform();
|
||||
let mut jre_entries = Vec::new();
|
||||
let mut jres = std::collections::BTreeMap::<String, (String, Option<ArtifactFormat>)>::new();
|
||||
let mut drivers = std::collections::BTreeMap::<String, (String, bool)>::new();
|
||||
|
||||
for index in 0..archive.len() {
|
||||
@@ -1608,11 +1987,21 @@ fn collect_offline_entries(
|
||||
return Err(format!("Offline package contains an unsafe path: {}", entry.name()));
|
||||
};
|
||||
let name = path.to_string_lossy().replace('\\', "/");
|
||||
if name.starts_with("jre/") && name.ends_with(".tar.gz") && name.contains(platform) {
|
||||
if name.starts_with("jre/") && name.contains(platform) {
|
||||
let jre_format = if name.ends_with(".tar.zst") {
|
||||
Some(ArtifactFormat::TarZstd)
|
||||
} else if name.ends_with(".tar.gz") {
|
||||
None
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
let jre_key = extract_jre_key_from_filename(&name)
|
||||
.ok_or_else(|| format!("Invalid JRE filename in offline package: {name}"))?;
|
||||
validate_offline_identifier(&jre_key, "JRE")?;
|
||||
jre_entries.push((jre_key, name));
|
||||
let replace = !jres.contains_key(&jre_key) || jre_format == Some(ArtifactFormat::TarZstd);
|
||||
if replace {
|
||||
jres.insert(jre_key, (name, jre_format));
|
||||
}
|
||||
} else if name.starts_with("drivers/") && name.ends_with(".jar") {
|
||||
let db_type = db_type_for_jar_offline_entry(registry, &name)
|
||||
.or_else(|| extract_db_type_from_filename(&name))
|
||||
@@ -1629,7 +2018,10 @@ fn collect_offline_entries(
|
||||
}
|
||||
}
|
||||
|
||||
Ok((jre_entries, drivers.into_iter().map(|(db_type, (name, is_native))| (db_type, name, is_native)).collect()))
|
||||
Ok((
|
||||
jres.into_iter().map(|(jre_key, (name, format))| (jre_key, name, format)).collect(),
|
||||
drivers.into_iter().map(|(db_type, (name, is_native))| (db_type, name, is_native)).collect(),
|
||||
))
|
||||
}
|
||||
|
||||
fn validate_offline_driver_entries(
|
||||
@@ -1723,7 +2115,36 @@ fn db_type_for_jar_offline_entry(registry: &AgentRegistry, name: &str) -> Option
|
||||
})
|
||||
}
|
||||
|
||||
fn jre_archive_suffix(format: Option<ArtifactFormat>) -> &'static str {
|
||||
match format {
|
||||
Some(ArtifactFormat::TarZstd) => ".tar.zst",
|
||||
None => ".tar.gz",
|
||||
}
|
||||
}
|
||||
|
||||
fn jre_archive_download_path(am: &AgentManager, jre_key: &str, format: Option<ArtifactFormat>) -> PathBuf {
|
||||
am.base_dir().join(format!("jre-{jre_key}-download{}", jre_archive_suffix(format)))
|
||||
}
|
||||
|
||||
fn extract_jre_archive(archive: &Path, dest: &Path, format: Option<ArtifactFormat>) -> Result<(), String> {
|
||||
match format {
|
||||
Some(ArtifactFormat::TarZstd) => {
|
||||
let file = std::fs::File::open(archive).map_err(|e| format!("Failed to open JRE archive: {e}"))?;
|
||||
let decoder =
|
||||
zstd::stream::read::Decoder::new(file).map_err(|e| format!("Failed to open zstd JRE archive: {e}"))?;
|
||||
extract_jre_tar(tar::Archive::new(decoder), dest)
|
||||
}
|
||||
None => extract_tar_gz(archive, dest),
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_tar_gz(archive: &Path, dest: &Path) -> Result<(), String> {
|
||||
let file = std::fs::File::open(archive).map_err(|e| format!("Failed to open JRE archive: {e}"))?;
|
||||
let decoder = flate2::read::GzDecoder::new(file);
|
||||
extract_jre_tar(tar::Archive::new(decoder), dest)
|
||||
}
|
||||
|
||||
fn extract_jre_tar<R: Read>(mut archive: tar::Archive<R>, dest: &Path) -> Result<(), String> {
|
||||
let parent = dest.parent().ok_or_else(|| format!("Invalid JRE destination: {}", dest.display()))?;
|
||||
std::fs::create_dir_all(parent).map_err(|e| format!("Failed to create JRE directory: {e}"))?;
|
||||
|
||||
@@ -1731,9 +2152,7 @@ fn extract_tar_gz(archive: &Path, dest: &Path) -> Result<(), String> {
|
||||
.prefix(".jre-extract-")
|
||||
.tempdir_in(parent)
|
||||
.map_err(|e| format!("Failed to create JRE extraction directory: {e}"))?;
|
||||
let file = std::fs::File::open(archive).map_err(|e| format!("Failed to open JRE archive: {e}"))?;
|
||||
let decoder = flate2::read::GzDecoder::new(file);
|
||||
tar::Archive::new(decoder).unpack(staging.path()).map_err(|e| format!("Failed to extract JRE archive: {e}"))?;
|
||||
archive.unpack(staging.path()).map_err(|e| format!("Failed to extract JRE archive: {e}"))?;
|
||||
|
||||
let mut roots = std::fs::read_dir(staging.path())
|
||||
.map_err(|e| format!("Failed to inspect extracted JRE archive: {e}"))?
|
||||
@@ -2090,7 +2509,7 @@ mod agent_download_url_tests {
|
||||
#[cfg(test)]
|
||||
mod agent_registry_install_tests {
|
||||
use super::*;
|
||||
use crate::agent_manager::{ArtifactInfo, DriverInfo, InstalledDriver, JavaRuntimeConfig, JreInfo};
|
||||
use crate::agent_manager::{ArtifactFormat, ArtifactInfo, DriverInfo, InstalledDriver, JavaRuntimeConfig, JreInfo};
|
||||
|
||||
fn test_manager(name: &str) -> AgentManager {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-agent-registry-install-{name}-{}", uuid::Uuid::new_v4()));
|
||||
@@ -2124,10 +2543,11 @@ mod agent_registry_install_tests {
|
||||
jar: Some(ArtifactInfo {
|
||||
url: format!("https://example.com/dbx-agent-{db_type}-legacy-placeholder.jar"),
|
||||
size: 0,
|
||||
format: None,
|
||||
}),
|
||||
native: [(
|
||||
AgentManager::current_platform().to_string(),
|
||||
ArtifactInfo { url: native_url.to_string(), size: native_size },
|
||||
ArtifactInfo { url: native_url.to_string(), size: native_size, format: None },
|
||||
)]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
@@ -2145,7 +2565,7 @@ mod agent_registry_install_tests {
|
||||
label: db_type.to_string(),
|
||||
min_app_version: "0.1.0".to_string(),
|
||||
jre: DEFAULT_JRE_KEY.to_string(),
|
||||
jar: Some(ArtifactInfo { url: url.to_string(), size }),
|
||||
jar: Some(ArtifactInfo { url: url.to_string(), size, format: None }),
|
||||
native: std::collections::HashMap::new(),
|
||||
},
|
||||
);
|
||||
@@ -2180,6 +2600,110 @@ mod agent_registry_install_tests {
|
||||
cache_path
|
||||
}
|
||||
|
||||
fn current_platform_native_binary() -> Vec<u8> {
|
||||
if cfg!(windows) {
|
||||
let mut bytes = vec![0_u8; 0x48];
|
||||
bytes[..2].copy_from_slice(b"MZ");
|
||||
bytes[0x3c..0x40].copy_from_slice(&(0x40_u32).to_le_bytes());
|
||||
bytes[0x40..0x44].copy_from_slice(b"PE\0\0");
|
||||
let machine = if cfg!(target_arch = "aarch64") { 0xaa64_u16 } else { 0x8664_u16 };
|
||||
bytes[0x44..0x46].copy_from_slice(&machine.to_le_bytes());
|
||||
bytes
|
||||
} else if cfg!(target_os = "linux") {
|
||||
let mut bytes = vec![0_u8; 20];
|
||||
bytes[..4].copy_from_slice(b"\x7fELF");
|
||||
bytes[4] = 2;
|
||||
bytes[5] = 1;
|
||||
let machine = if cfg!(target_arch = "aarch64") { 183_u16 } else { 62_u16 };
|
||||
bytes[18..20].copy_from_slice(&machine.to_le_bytes());
|
||||
bytes
|
||||
} else if cfg!(target_os = "macos") {
|
||||
let mut bytes = vec![0xcf, 0xfa, 0xed, 0xfe];
|
||||
let cpu_type = if cfg!(target_arch = "aarch64") { 0x0100_000c_u32 } else { 0x0100_0007_u32 };
|
||||
bytes.extend_from_slice(&cpu_type.to_le_bytes());
|
||||
bytes
|
||||
} else {
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
|
||||
fn test_agent_jar() -> Vec<u8> {
|
||||
let mut bytes = std::io::Cursor::new(Vec::new());
|
||||
{
|
||||
use std::io::Write;
|
||||
let mut archive = zip::ZipWriter::new(&mut bytes);
|
||||
archive.start_file("META-INF/MANIFEST.MF", zip::write::SimpleFileOptions::default()).unwrap();
|
||||
archive.write_all(b"Manifest-Version: 1.0\nMain-Class: com.dbx.Agent\n").unwrap();
|
||||
archive.finish().unwrap();
|
||||
}
|
||||
bytes.into_inner()
|
||||
}
|
||||
|
||||
fn build_tar_zstd_driver_package(
|
||||
db_type: &str,
|
||||
version: &str,
|
||||
kind: DriverArtifactKind,
|
||||
driver_bytes: &[u8],
|
||||
) -> Vec<u8> {
|
||||
let platform = AgentManager::current_platform();
|
||||
let (filename, artifact) = match kind {
|
||||
DriverArtifactKind::Jar => (
|
||||
format!("dbx-agent-{db_type}-{version}.jar"),
|
||||
serde_json::json!({
|
||||
"jar": {
|
||||
"url": format!("dbx-agent-{db_type}-{version}.jar"),
|
||||
"size": driver_bytes.len()
|
||||
}
|
||||
}),
|
||||
),
|
||||
DriverArtifactKind::Native => {
|
||||
let extension = if platform.starts_with("windows-") { ".exe" } else { "" };
|
||||
let filename = format!("dbx-agent-{db_type}-{version}-{platform}{extension}");
|
||||
(
|
||||
filename.clone(),
|
||||
serde_json::json!({
|
||||
"native": {
|
||||
platform: {
|
||||
"url": filename,
|
||||
"size": driver_bytes.len()
|
||||
}
|
||||
}
|
||||
}),
|
||||
)
|
||||
}
|
||||
};
|
||||
let mut driver = serde_json::json!({
|
||||
"version": version,
|
||||
"label": db_type,
|
||||
"min_app_version": "0.6.0",
|
||||
"jre": DEFAULT_JRE_KEY
|
||||
});
|
||||
driver.as_object_mut().unwrap().extend(artifact.as_object().unwrap().clone());
|
||||
let registry = serde_json::json!({
|
||||
"jres": {},
|
||||
"drivers": {
|
||||
db_type: driver
|
||||
}
|
||||
});
|
||||
let registry_bytes = registry.to_string().into_bytes();
|
||||
let encoder = zstd::stream::write::Encoder::new(Vec::new(), 3).unwrap();
|
||||
let mut archive = tar::Builder::new(encoder);
|
||||
|
||||
let mut registry_header = tar::Header::new_gnu();
|
||||
registry_header.set_size(registry_bytes.len() as u64);
|
||||
registry_header.set_mode(0o644);
|
||||
registry_header.set_cksum();
|
||||
archive.append_data(&mut registry_header, "agent-registry.json", registry_bytes.as_slice()).unwrap();
|
||||
|
||||
let mut driver_header = tar::Header::new_gnu();
|
||||
driver_header.set_size(driver_bytes.len() as u64);
|
||||
driver_header.set_mode(if kind == DriverArtifactKind::Native { 0o755 } else { 0o644 });
|
||||
driver_header.set_cksum();
|
||||
archive.append_data(&mut driver_header, format!("drivers/{filename}"), driver_bytes).unwrap();
|
||||
|
||||
archive.into_inner().unwrap().finish().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn managed_jre_content_revision_triggers_install() {
|
||||
let manager = test_manager("jre-content-revision");
|
||||
@@ -2208,7 +2732,7 @@ mod agent_registry_install_tests {
|
||||
version: version.to_string(),
|
||||
platforms: [(
|
||||
AgentManager::current_platform().to_string(),
|
||||
ArtifactInfo { url: url.to_string(), size },
|
||||
ArtifactInfo { url: url.to_string(), size, format: None },
|
||||
)]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
@@ -2234,8 +2758,29 @@ mod agent_registry_install_tests {
|
||||
builder.into_inner().unwrap().finish().unwrap()
|
||||
}
|
||||
|
||||
fn write_cached_jre_download(am: &AgentManager, jre_key: &str, version: &str, url: &str, archive: &[u8]) {
|
||||
let dest = am.base_dir().join(format!("jre-{jre_key}-download.tar.gz"));
|
||||
fn build_zstd_jre_archive(am: &AgentManager, jre_key: &str) -> Vec<u8> {
|
||||
let archive_root = am.base_dir().join("jre-zstd-test-archive");
|
||||
let payload = archive_root.join("payload");
|
||||
let java_path = am.jre_java_path(jre_key);
|
||||
let relative_java_path = java_path.strip_prefix(am.jre_dir(jre_key)).unwrap();
|
||||
let java_path = payload.join(relative_java_path);
|
||||
std::fs::create_dir_all(java_path.parent().unwrap()).unwrap();
|
||||
std::fs::write(java_path, b"java").unwrap();
|
||||
let encoder = zstd::stream::write::Encoder::new(Vec::new(), 3).unwrap();
|
||||
let mut builder = tar::Builder::new(encoder);
|
||||
builder.append_dir_all("payload", &payload).unwrap();
|
||||
builder.into_inner().unwrap().finish().unwrap()
|
||||
}
|
||||
|
||||
fn write_cached_jre_download(
|
||||
am: &AgentManager,
|
||||
jre_key: &str,
|
||||
version: &str,
|
||||
url: &str,
|
||||
format: Option<ArtifactFormat>,
|
||||
archive: &[u8],
|
||||
) {
|
||||
let dest = jre_archive_download_path(am, jre_key, format);
|
||||
let cache_path = cached_download_path(
|
||||
am,
|
||||
url,
|
||||
@@ -2284,6 +2829,81 @@ mod agent_registry_install_tests {
|
||||
.any(|event| event.step == "done" && event.db_type.as_deref() == Some(db_type)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registry_install_extracts_tar_zstd_native_driver_package() {
|
||||
let manager = test_manager("tar-zstd-native-package");
|
||||
let db_type = "duckdb";
|
||||
let version = "0.1.0";
|
||||
let package_url = "https://example.com/dbx-agent-duckdb.tar.zst";
|
||||
let native_bytes = current_platform_native_binary();
|
||||
let package_bytes = build_tar_zstd_driver_package(db_type, version, DriverArtifactKind::Native, &native_bytes);
|
||||
let mut registry =
|
||||
registry_with_native_and_legacy_jar(db_type, version, package_url, package_bytes.len() as u64);
|
||||
registry.drivers.get_mut(db_type).unwrap().native.get_mut(AgentManager::current_platform()).unwrap().format =
|
||||
Some(ArtifactFormat::TarZstd);
|
||||
let native_path = manager.driver_native_path(db_type);
|
||||
let package_path = driver_artifact_download_path(&native_path, Some(ArtifactFormat::TarZstd));
|
||||
let cache_path =
|
||||
write_cached_driver_download(&manager, db_type, version, package_url, &package_path, &package_bytes);
|
||||
let progress = |_| {};
|
||||
|
||||
install_agent_driver_from_registry(
|
||||
&manager,
|
||||
®istry,
|
||||
DownloadSource::Official,
|
||||
db_type,
|
||||
&progress,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(std::fs::read(&native_path).unwrap(), native_bytes);
|
||||
assert!(!package_path.exists());
|
||||
assert!(!cache_path.exists());
|
||||
assert_eq!(manager.load_state().installed_drivers[db_type].version, version);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registry_install_extracts_tar_zstd_java_driver_package() {
|
||||
let manager = test_manager("tar-zstd-java-package");
|
||||
let db_type = "dameng";
|
||||
let version = "0.2.0";
|
||||
let package_url = "https://example.com/dbx-agent-dameng.tar.zst";
|
||||
let jar_bytes = test_agent_jar();
|
||||
let package_bytes = build_tar_zstd_driver_package(db_type, version, DriverArtifactKind::Jar, &jar_bytes);
|
||||
let mut registry = registry_with_jar(db_type, version, package_url, package_bytes.len() as u64);
|
||||
registry.drivers.get_mut(db_type).unwrap().jar.as_mut().unwrap().format = Some(ArtifactFormat::TarZstd);
|
||||
manager
|
||||
.mutate_state(|state| {
|
||||
state.java_runtime = JavaRuntimeConfig { mode: JavaRuntimeMode::System, custom_java_path: None };
|
||||
})
|
||||
.unwrap();
|
||||
let jar_path = manager.driver_jar_path(db_type);
|
||||
let package_path = driver_artifact_download_path(&jar_path, Some(ArtifactFormat::TarZstd));
|
||||
let cache_path =
|
||||
write_cached_driver_download(&manager, db_type, version, package_url, &package_path, &package_bytes);
|
||||
let progress = |_| {};
|
||||
|
||||
install_agent_driver_from_registry(
|
||||
&manager,
|
||||
®istry,
|
||||
DownloadSource::Official,
|
||||
db_type,
|
||||
&progress,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(std::fs::read(&jar_path).unwrap(), jar_bytes);
|
||||
assert!(!package_path.exists());
|
||||
assert!(!cache_path.exists());
|
||||
assert_eq!(manager.load_state().installed_drivers[db_type].version, version);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn batch_upgrade_preserves_successful_state_and_reports_independent_failure() {
|
||||
let manager = test_manager("batch-upgrade");
|
||||
@@ -2367,7 +2987,7 @@ mod agent_registry_install_tests {
|
||||
let events = std::sync::Mutex::new(Vec::new());
|
||||
let progress = |event| events.lock().unwrap().push(event);
|
||||
|
||||
write_cached_jre_download(&manager, jre_key, version, url, &archive);
|
||||
write_cached_jre_download(&manager, jre_key, version, url, None, &archive);
|
||||
ensure_jre_from_registry(
|
||||
&manager,
|
||||
®istry,
|
||||
@@ -2383,7 +3003,7 @@ mod agent_registry_install_tests {
|
||||
|
||||
// The successful install cleans its cache. Re-add it so the old
|
||||
// implementation fails deterministically by consuming it again.
|
||||
write_cached_jre_download(&manager, jre_key, version, url, &archive);
|
||||
write_cached_jre_download(&manager, jre_key, version, url, None, &archive);
|
||||
ensure_jre_from_registry(
|
||||
&manager,
|
||||
®istry,
|
||||
@@ -2410,6 +3030,26 @@ mod agent_registry_install_tests {
|
||||
.any(|event| event.step == "jre" && event.db_type.as_deref() == Some("dameng")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn managed_jre_install_extracts_tar_zstd_archive() {
|
||||
let manager = test_manager("managed-zstd-jre");
|
||||
let jre_key = DEFAULT_JRE_KEY;
|
||||
let version = "21.0.12";
|
||||
let url = "https://example.com/dbx-jre.tar.zst";
|
||||
let archive = build_zstd_jre_archive(&manager, jre_key);
|
||||
let mut registry = registry_with_jre(jre_key, version, url, archive.len() as u64);
|
||||
registry.jres.get_mut(jre_key).unwrap().platforms.get_mut(AgentManager::current_platform()).unwrap().format =
|
||||
Some(ArtifactFormat::TarZstd);
|
||||
write_cached_jre_download(&manager, jre_key, version, url, Some(ArtifactFormat::TarZstd), &archive);
|
||||
|
||||
ensure_jre_from_registry(&manager, ®istry, DownloadSource::Official, jre_key, "dameng", &|_| {}, None, None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(manager.is_jre_installed(jre_key));
|
||||
assert_eq!(manager.load_state().jre_versions[jre_key], version);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stash_is_recorded_before_jre_extraction() {
|
||||
let manager = test_manager("stash-before-extract");
|
||||
|
||||
@@ -4,8 +4,6 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::{watch, Mutex, RwLock};
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
use mysql_async::prelude::Queryable;
|
||||
use mysql_async::Row as MysqlRow;
|
||||
@@ -42,21 +40,16 @@ const ACCESS_AGENT_CONNECT_TIMEOUT_SECS: u64 = 30;
|
||||
const POOL_CLOSE_TIMEOUT_SECS: u64 = 3;
|
||||
const HEALTH_CHECK_POOL_ACQUIRE_TIMEOUT: Duration = Duration::from_millis(500);
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
mod duckdb_types {
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
use std::sync::Arc;
|
||||
pub type DuckDbHandle = Arc<crate::db::duckdb_driver::DuckDbConnection>;
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
pub type DuckDbWorkerHandle = Arc<crate::db::duckdb_worker_process::DuckDbWorkerClient>;
|
||||
pub type ExternalTabularHandle = Arc<crate::external::ExternalPool>;
|
||||
}
|
||||
#[cfg(not(feature = "duckdb-bundled"))]
|
||||
mod duckdb_types {
|
||||
pub type DuckDbHandle = ();
|
||||
#[cfg(not(feature = "duckdb-sidecar"))]
|
||||
pub type DuckDbWorkerHandle = ();
|
||||
pub type ExternalTabularHandle = ();
|
||||
}
|
||||
|
||||
use duckdb_types::{DuckDbHandle, DuckDbWorkerHandle, ExternalTabularHandle};
|
||||
use duckdb_types::DuckDbWorkerHandle;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub enum MysqlMode {
|
||||
@@ -100,7 +93,6 @@ pub enum PoolKind {
|
||||
Turso(db::turso_driver::TursoClient),
|
||||
CloudflareD1(db::cloudflare_d1_driver::CloudflareD1Client),
|
||||
Redis(db::redis_driver::RedisConnection),
|
||||
DuckDb(DuckDbHandle),
|
||||
DuckDbWorker(DuckDbWorkerHandle),
|
||||
MongoDb(mongodb::Client),
|
||||
ClickHouse(db::clickhouse_driver::ChClient),
|
||||
@@ -110,7 +102,6 @@ pub enum PoolKind {
|
||||
VectorDb(db::vector_driver::VectorClient),
|
||||
InfluxDb(db::influxdb_driver::InfluxdbClient),
|
||||
Agent(Arc<tokio::sync::Mutex<db::agent_driver::AgentDriverClient>>),
|
||||
ExternalTabular(ExternalTabularHandle),
|
||||
ExternalDriver {
|
||||
driver_id: String,
|
||||
config: Arc<ConnectionConfig>,
|
||||
@@ -690,41 +681,44 @@ impl AppState {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
async fn create_duckdb_pool(&self, config: &ConnectionConfig) -> Result<PoolKind, String> {
|
||||
if self.duckdb_worker_process_isolation.load(Ordering::Relaxed) {
|
||||
let attached_databases = config
|
||||
.attached_databases
|
||||
.iter()
|
||||
.map(|attached| crate::models::connection::AttachedDatabaseConfig {
|
||||
name: attached.name.clone(),
|
||||
path: expand_tilde(&attached.path),
|
||||
})
|
||||
.collect();
|
||||
let client = db::duckdb_worker_process::DuckDbWorkerClient::open_with_process_limit(
|
||||
expand_tilde(&config.host),
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
pub async fn create_duckdb_pool(&self, config: &ConnectionConfig) -> Result<PoolKind, String> {
|
||||
let attached_databases = config
|
||||
.attached_databases
|
||||
.iter()
|
||||
.map(|attached| crate::models::connection::AttachedDatabaseConfig {
|
||||
name: attached.name.clone(),
|
||||
path: expand_tilde(&attached.path),
|
||||
})
|
||||
.collect();
|
||||
let path = expand_tilde(&config.host);
|
||||
let init_script = config.init_script.clone();
|
||||
let process_limit = self.duckdb_worker_max_processes.load(Ordering::Relaxed);
|
||||
let installed_driver = self.agent_manager.driver_native_path("duckdb");
|
||||
let has_driver_override =
|
||||
std::env::var_os(db::duckdb_worker_process::DUCKDB_DRIVER_PATH_ENV).is_some_and(|value| !value.is_empty());
|
||||
let client = if installed_driver.is_file() && !has_driver_override {
|
||||
db::duckdb_worker_process::DuckDbWorkerClient::open_with_executable_and_process_limit(
|
||||
installed_driver,
|
||||
path,
|
||||
attached_databases,
|
||||
config.init_script.clone(),
|
||||
self.duckdb_worker_max_processes.load(Ordering::Relaxed),
|
||||
init_script,
|
||||
process_limit,
|
||||
)
|
||||
.await?;
|
||||
Ok(PoolKind::DuckDbWorker(Arc::new(client)))
|
||||
.await?
|
||||
} else {
|
||||
let con = db::duckdb_driver::connect_path(&expand_tilde(&config.host))?;
|
||||
{
|
||||
let locked = con.lock().map_err(|e| e.to_string())?;
|
||||
for attached in &config.attached_databases {
|
||||
crate::schema::duckdb_attach_database(&locked, &attached.name, &expand_tilde(&attached.path))?;
|
||||
}
|
||||
if let Some(script) = config.init_script.as_deref() {
|
||||
db::duckdb_driver::run_init_script(&locked, script)?;
|
||||
}
|
||||
}
|
||||
Ok(PoolKind::DuckDb(con))
|
||||
}
|
||||
db::duckdb_worker_process::DuckDbWorkerClient::open_with_process_limit(
|
||||
path,
|
||||
attached_databases,
|
||||
init_script,
|
||||
process_limit,
|
||||
)
|
||||
.await?
|
||||
};
|
||||
Ok(PoolKind::DuckDbWorker(Arc::new(client)))
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
pub async fn test_duckdb_connection_config(&self, config: &ConnectionConfig) -> Result<(), String> {
|
||||
// Test the submitted form as a fresh session so unsaved ATTACH/init
|
||||
// changes cannot be masked by a pool created from older settings.
|
||||
@@ -1315,11 +1309,11 @@ impl AppState {
|
||||
};
|
||||
PoolKind::Redis(con)
|
||||
}
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
DatabaseType::DuckDb => self.create_duckdb_pool(&db_config).await?,
|
||||
#[cfg(not(feature = "duckdb-bundled"))]
|
||||
#[cfg(not(feature = "duckdb-sidecar"))]
|
||||
DatabaseType::DuckDb => {
|
||||
return Err("DuckDB support is not compiled in this build. Rebuild with default features.".to_string());
|
||||
return Err("DuckDB support is not compiled in this build.".to_string());
|
||||
}
|
||||
DatabaseType::MongoDb => {
|
||||
if mongo_uses_legacy_driver(&db_config) {
|
||||
@@ -2342,9 +2336,7 @@ impl AppState {
|
||||
}
|
||||
}
|
||||
PoolKind::Sqlite(_)
|
||||
| PoolKind::DuckDb(_)
|
||||
| PoolKind::DuckDbWorker(_)
|
||||
| PoolKind::ExternalTabular(_)
|
||||
| PoolKind::ExternalDriver { .. }
|
||||
| PoolKind::MessageQueue
|
||||
| PoolKind::Nacos => false,
|
||||
@@ -2567,123 +2559,10 @@ impl AppState {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
async fn remove_pool_by_key_detached(&self, pool_key: &str) -> bool {
|
||||
self.stop_keepalive_task(pool_key).await;
|
||||
self.pool_activity.write().await.remove(pool_key);
|
||||
self.postgres_cancel_contexts.write().await.remove(pool_key);
|
||||
let removed = self.connections.write().await.remove(pool_key);
|
||||
if let Some(pool) = removed {
|
||||
close_removed_pools_in_background(&self.task_supervisor, vec![(pool_key.to_string(), pool)]);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
async fn remove_pool_if_duckdb_isolation_mismatch(&self, pool_key: &str) -> bool {
|
||||
let isolation_enabled = self.duckdb_worker_process_isolation.load(Ordering::Relaxed);
|
||||
let mismatch = {
|
||||
let connections = self.connections.read().await;
|
||||
match connections.get(pool_key) {
|
||||
Some(PoolKind::DuckDb(_)) => isolation_enabled,
|
||||
Some(PoolKind::DuckDbWorker(_)) => !isolation_enabled,
|
||||
_ => false,
|
||||
}
|
||||
};
|
||||
if mismatch {
|
||||
self.remove_pool_by_key_detached(pool_key).await
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "duckdb-bundled"))]
|
||||
async fn remove_pool_if_duckdb_isolation_mismatch(&self, _pool_key: &str) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
pub fn spawn_duckdb_pool_cleanup(&self, pool_key: String, con: DuckDbHandle) {
|
||||
let connections = self.connections.clone();
|
||||
let supervisor = self.task_supervisor.clone();
|
||||
let pool_activity = self.pool_activity.clone();
|
||||
let postgres_cancel_contexts = self.postgres_cancel_contexts.clone();
|
||||
let task_key = format!("duckdb-cleanup:{pool_key}");
|
||||
supervisor.clone().spawn_replace(task_key, move |shutdown| async move {
|
||||
while Arc::strong_count(&con) > 2 {
|
||||
tokio::select! {
|
||||
_ = shutdown.cancelled() => return,
|
||||
_ = tokio::time::sleep(Duration::from_millis(50)) => {}
|
||||
}
|
||||
}
|
||||
supervisor.stop(&format!("keepalive:{pool_key}"));
|
||||
pool_activity.write().await.remove(&pool_key);
|
||||
postgres_cancel_contexts.write().await.remove(&pool_key);
|
||||
let removed = {
|
||||
let mut conns = connections.write().await;
|
||||
match conns.get(&pool_key) {
|
||||
Some(PoolKind::DuckDb(current)) if Arc::ptr_eq(current, &con) => conns.remove(&pool_key),
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
if let Some(pool) = removed {
|
||||
// Keep the old DuckDB pool marked as draining until it is no longer
|
||||
// visible in the pool map, otherwise a concurrent query could reuse it.
|
||||
con.clear_draining();
|
||||
drop(con);
|
||||
close_pool_kind_with_timeout(pool_key, pool).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
pub fn spawn_duckdb_draining_cleanup(
|
||||
&self,
|
||||
pool_key: String,
|
||||
con: DuckDbHandle,
|
||||
mut task: JoinHandle<Result<db::QueryResult, String>>,
|
||||
) {
|
||||
let connections = self.connections.clone();
|
||||
let supervisor = self.task_supervisor.clone();
|
||||
let pool_activity = self.pool_activity.clone();
|
||||
let postgres_cancel_contexts = self.postgres_cancel_contexts.clone();
|
||||
let task_key = format!("duckdb-draining:{pool_key}");
|
||||
supervisor.clone().spawn_replace(task_key, move |shutdown| async move {
|
||||
tokio::select! {
|
||||
_ = shutdown.cancelled() => {
|
||||
task.abort();
|
||||
return;
|
||||
}
|
||||
_ = &mut task => {}
|
||||
}
|
||||
while Arc::strong_count(&con) > 2 {
|
||||
tokio::select! {
|
||||
_ = shutdown.cancelled() => return,
|
||||
_ = tokio::time::sleep(Duration::from_millis(50)) => {}
|
||||
}
|
||||
}
|
||||
supervisor.stop(&format!("keepalive:{pool_key}"));
|
||||
pool_activity.write().await.remove(&pool_key);
|
||||
postgres_cancel_contexts.write().await.remove(&pool_key);
|
||||
let removed = {
|
||||
let mut conns = connections.write().await;
|
||||
match conns.get(&pool_key) {
|
||||
Some(PoolKind::DuckDb(current)) if Arc::ptr_eq(current, &con) => conns.remove(&pool_key),
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
if let Some(pool) = removed {
|
||||
// Keep the old DuckDB pool marked as draining until it is no longer
|
||||
// visible in the pool map, otherwise a concurrent query could reuse it.
|
||||
con.clear_draining();
|
||||
drop(con);
|
||||
close_pool_kind_with_timeout(pool_key, pool).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub async fn close_database_pool(&self, connection_id: &str, database: Option<&str>) -> Result<bool, String> {
|
||||
let db_type = {
|
||||
let configs = self.configs.read().await;
|
||||
@@ -2732,6 +2611,11 @@ impl AppState {
|
||||
let mut keys = HashSet::new();
|
||||
|
||||
for (pool_key, pool) in connections.iter() {
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
if matches!(pool, PoolKind::DuckDbWorker(_)) {
|
||||
keys.insert("duckdb".to_string());
|
||||
continue;
|
||||
}
|
||||
if !matches!(pool, PoolKind::Agent(_)) {
|
||||
continue;
|
||||
}
|
||||
@@ -3092,9 +2976,7 @@ impl AppState {
|
||||
}
|
||||
}
|
||||
PoolKind::Sqlite(_)
|
||||
| PoolKind::DuckDb(_)
|
||||
| PoolKind::DuckDbWorker(_)
|
||||
| PoolKind::ExternalTabular(_)
|
||||
| PoolKind::ExternalDriver { .. }
|
||||
| PoolKind::MessageQueue
|
||||
| PoolKind::Nacos => true,
|
||||
@@ -3162,13 +3044,13 @@ impl AppState {
|
||||
close_removed_pools_in_background(&self.task_supervisor, removed);
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
async fn remove_duckdb_pools_detached(&self) {
|
||||
let removed = self.drain_duckdb_pools().await;
|
||||
close_removed_pools_in_background(&self.task_supervisor, removed);
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "duckdb-bundled"))]
|
||||
#[cfg(not(feature = "duckdb-sidecar"))]
|
||||
async fn remove_duckdb_pools_detached(&self) {}
|
||||
|
||||
pub async fn remove_external_driver_pools(&self, driver_id: &str) {
|
||||
@@ -3209,16 +3091,16 @@ impl AppState {
|
||||
removed
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
async fn drain_duckdb_pools(&self) -> Vec<(String, PoolKind)> {
|
||||
let keys_to_remove: Vec<String> = self
|
||||
.connections
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.filter_map(|(key, pool)| match pool {
|
||||
PoolKind::DuckDb(_) | PoolKind::DuckDbWorker(_) => Some(key.clone()),
|
||||
_ => None,
|
||||
.filter_map(|(key, pool)| {
|
||||
let is_duckdb = matches!(pool, PoolKind::DuckDbWorker(_));
|
||||
is_duckdb.then(|| key.clone())
|
||||
})
|
||||
.collect();
|
||||
self.stop_keepalive_tasks(&keys_to_remove).await;
|
||||
@@ -3516,13 +3398,9 @@ fn clone_pool_kind(pool: &PoolKind) -> PoolKind {
|
||||
PoolKind::Rqlite(client) => PoolKind::Rqlite(client.clone()),
|
||||
PoolKind::Turso(client) => PoolKind::Turso(client.clone()),
|
||||
PoolKind::CloudflareD1(client) => PoolKind::CloudflareD1(client.clone()),
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
PoolKind::DuckDb(con) => PoolKind::DuckDb(con.clone()),
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
PoolKind::DuckDbWorker(client) => PoolKind::DuckDbWorker(client.clone()),
|
||||
#[cfg(not(feature = "duckdb-bundled"))]
|
||||
PoolKind::DuckDb(_) => PoolKind::DuckDb(()),
|
||||
#[cfg(not(feature = "duckdb-bundled"))]
|
||||
#[cfg(not(feature = "duckdb-sidecar"))]
|
||||
PoolKind::DuckDbWorker(_) => PoolKind::DuckDbWorker(()),
|
||||
PoolKind::MongoDb(client) => PoolKind::MongoDb(client.clone()),
|
||||
PoolKind::ClickHouse(client) => PoolKind::ClickHouse(client.clone()),
|
||||
@@ -3532,10 +3410,6 @@ fn clone_pool_kind(pool: &PoolKind) -> PoolKind {
|
||||
PoolKind::VectorDb(client) => PoolKind::VectorDb(client.clone()),
|
||||
PoolKind::InfluxDb(client) => PoolKind::InfluxDb(client.clone()),
|
||||
PoolKind::Agent(client) => PoolKind::Agent(client.clone()),
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
PoolKind::ExternalTabular(ext) => PoolKind::ExternalTabular(ext.clone()),
|
||||
#[cfg(not(feature = "duckdb-bundled"))]
|
||||
PoolKind::ExternalTabular(_) => PoolKind::ExternalTabular(()),
|
||||
PoolKind::ExternalDriver { driver_id, config, session } => {
|
||||
PoolKind::ExternalDriver { driver_id: driver_id.clone(), config: config.clone(), session: session.clone() }
|
||||
}
|
||||
@@ -3558,17 +3432,11 @@ pub async fn close_pool_kind(pool: PoolKind) {
|
||||
PoolKind::Redis(conn) => {
|
||||
drop(conn);
|
||||
}
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
PoolKind::DuckDb(con) => {
|
||||
crate::db::duckdb_driver::close_connection(con);
|
||||
}
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
PoolKind::DuckDbWorker(client) => {
|
||||
client.shutdown().await;
|
||||
}
|
||||
#[cfg(not(feature = "duckdb-bundled"))]
|
||||
PoolKind::DuckDb(_) => {}
|
||||
#[cfg(not(feature = "duckdb-bundled"))]
|
||||
#[cfg(not(feature = "duckdb-sidecar"))]
|
||||
PoolKind::DuckDbWorker(_) => {}
|
||||
PoolKind::MongoDb(client) => {
|
||||
drop(client);
|
||||
@@ -3595,7 +3463,6 @@ pub async fn close_pool_kind(pool: PoolKind) {
|
||||
let mut client = client.lock().await;
|
||||
let _ = client.disconnect().await;
|
||||
}
|
||||
PoolKind::ExternalTabular(_) => {}
|
||||
PoolKind::ExternalDriver { session, .. } => {
|
||||
session.shutdown().await;
|
||||
}
|
||||
@@ -5108,51 +4975,6 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[tokio::test]
|
||||
async fn duckdb_connection_test_does_not_reuse_stale_pool_config() {
|
||||
let (state, dir) = test_app_state().await;
|
||||
let db_path = dir.join("app.duckdb");
|
||||
duckdb::Connection::open(&db_path).unwrap();
|
||||
let mut config = mysql_config(None);
|
||||
config.id = "duckdb-conn".to_string();
|
||||
config.name = "DuckDB".to_string();
|
||||
config.db_type = DatabaseType::DuckDb;
|
||||
config.host = db_path.to_string_lossy().to_string();
|
||||
config.port = 0;
|
||||
|
||||
state.configs.write().await.insert(config.id.clone(), config.clone());
|
||||
state.get_or_create_pool("duckdb-conn", None).await.unwrap();
|
||||
|
||||
config.init_script = Some("SELECT definitely_invalid_syntax(".to_string());
|
||||
let error = state.test_duckdb_connection_config(&config).await.expect_err("invalid submitted script must fail");
|
||||
|
||||
assert!(error.contains("Connection init script statement 1 failed"));
|
||||
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[tokio::test]
|
||||
async fn duckdb_connection_test_validates_attached_databases() {
|
||||
let (state, dir) = test_app_state().await;
|
||||
let mut config = mysql_config(None);
|
||||
config.id = "duckdb-conn".to_string();
|
||||
config.name = "DuckDB".to_string();
|
||||
config.db_type = DatabaseType::DuckDb;
|
||||
config.host = ":memory:".to_string();
|
||||
config.port = 0;
|
||||
config.attached_databases.push(AttachedDatabaseConfig {
|
||||
name: "missing".to_string(),
|
||||
path: dir.join("missing-parent").join("missing.duckdb").to_string_lossy().to_string(),
|
||||
});
|
||||
|
||||
let error = state.test_duckdb_connection_config(&config).await.expect_err("invalid attach must fail");
|
||||
|
||||
assert!(error.to_ascii_lowercase().contains("attach"));
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_connection_restores_attached_databases() {
|
||||
let (state, dir) = test_app_state().await;
|
||||
@@ -5189,72 +5011,6 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[tokio::test]
|
||||
async fn duckdb_connection_test_supports_memory_bootstrap() {
|
||||
let (state, dir) = test_app_state().await;
|
||||
let mut config = mysql_config(None);
|
||||
config.id = "duckdb-memory".to_string();
|
||||
config.name = "DuckDB memory".to_string();
|
||||
config.db_type = DatabaseType::DuckDb;
|
||||
config.host = ":memory:".to_string();
|
||||
config.port = 0;
|
||||
config.init_script = Some(r#"CREATE TABLE probe AS SELECT E'it\'s;ok' AS value;"#.to_string());
|
||||
|
||||
state.test_duckdb_connection_config(&config).await.expect("memory bootstrap succeeds");
|
||||
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[tokio::test]
|
||||
async fn applying_duckdb_worker_isolation_drops_existing_duckdb_pools() {
|
||||
let (state, dir) = test_app_state().await;
|
||||
let db_path = dir.join("app.duckdb");
|
||||
duckdb::Connection::open(&db_path).unwrap();
|
||||
let mut config = mysql_config(None);
|
||||
config.id = "duckdb-conn".to_string();
|
||||
config.name = "DuckDB".to_string();
|
||||
config.db_type = DatabaseType::DuckDb;
|
||||
config.host = db_path.to_string_lossy().to_string();
|
||||
config.port = 0;
|
||||
|
||||
state.configs.write().await.insert(config.id.clone(), config);
|
||||
state.get_or_create_pool("duckdb-conn", None).await.unwrap();
|
||||
assert!(matches!(state.connections.read().await.get("duckdb-conn"), Some(PoolKind::DuckDb(_))));
|
||||
|
||||
state.apply_duckdb_worker_process_isolation(true).await;
|
||||
|
||||
assert!(!state.connections.read().await.contains_key("duckdb-conn"));
|
||||
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[tokio::test]
|
||||
async fn duckdb_pool_mode_mismatch_removes_existing_pool() {
|
||||
let (state, dir) = test_app_state().await;
|
||||
let db_path = dir.join("app.duckdb");
|
||||
duckdb::Connection::open(&db_path).unwrap();
|
||||
let mut config = mysql_config(None);
|
||||
config.id = "duckdb-conn".to_string();
|
||||
config.name = "DuckDB".to_string();
|
||||
config.db_type = DatabaseType::DuckDb;
|
||||
config.host = db_path.to_string_lossy().to_string();
|
||||
config.port = 0;
|
||||
|
||||
state.configs.write().await.insert(config.id.clone(), config);
|
||||
state.get_or_create_pool("duckdb-conn", None).await.unwrap();
|
||||
assert!(matches!(state.connections.read().await.get("duckdb-conn"), Some(PoolKind::DuckDb(_))));
|
||||
|
||||
state.set_duckdb_worker_process_isolation_enabled(true);
|
||||
|
||||
assert!(state.remove_pool_if_duckdb_isolation_mismatch("duckdb-conn").await);
|
||||
assert!(!state.connections.read().await.contains_key("duckdb-conn"));
|
||||
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remove_connection_pools_clears_base_and_database_scoped_pools() {
|
||||
let (state, dir) = test_app_state().await;
|
||||
@@ -5519,33 +5275,6 @@ for line in sys.stdin:
|
||||
assert!(!super::is_agent_validate_connection_unsupported("Agent RPC error (-1): Unknown method: kv_put"));
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[tokio::test]
|
||||
async fn duckdb_client_session_reuses_base_pool_to_avoid_file_locks() {
|
||||
let (state, dir) = test_app_state().await;
|
||||
let db_path = dir.join("session.duckdb");
|
||||
duckdb::Connection::open(&db_path).unwrap();
|
||||
let mut config = mysql_config(None);
|
||||
config.id = "duckdb-conn".to_string();
|
||||
config.name = "DuckDB".to_string();
|
||||
config.db_type = DatabaseType::DuckDb;
|
||||
config.host = db_path.to_string_lossy().to_string();
|
||||
config.port = 0;
|
||||
|
||||
state.configs.write().await.insert(config.id.clone(), config.clone());
|
||||
let base_pool_key = state.get_or_create_pool("duckdb-conn", None).await.unwrap();
|
||||
let pool_key = state.get_or_create_pool_for_session("duckdb-conn", Some("main"), Some("tab-1")).await.unwrap();
|
||||
assert_eq!(pool_key, base_pool_key);
|
||||
|
||||
assert!(!state.close_client_session_pool("duckdb-conn", Some("main"), "tab-1").await.unwrap());
|
||||
|
||||
let conns = state.connections.read().await;
|
||||
assert!(conns.contains_key("duckdb-conn"));
|
||||
assert!(!conns.contains_key("duckdb-conn:session:tab-1"));
|
||||
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_memory_client_sessions_share_the_same_database() {
|
||||
let (state, dir) = test_app_state().await;
|
||||
|
||||
@@ -1,198 +0,0 @@
|
||||
#![cfg(feature = "duckdb-bundled")]
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
|
||||
pub struct DuckDbConnection {
|
||||
connection: Mutex<duckdb::Connection>,
|
||||
interrupt_handle: Arc<duckdb::InterruptHandle>,
|
||||
draining: AtomicBool,
|
||||
}
|
||||
|
||||
impl DuckDbConnection {
|
||||
pub fn new(connection: duckdb::Connection) -> Self {
|
||||
let interrupt_handle = connection.interrupt_handle();
|
||||
Self { connection: Mutex::new(connection), interrupt_handle, draining: AtomicBool::new(false) }
|
||||
}
|
||||
|
||||
pub fn lock(&self) -> std::sync::LockResult<std::sync::MutexGuard<'_, duckdb::Connection>> {
|
||||
self.connection.lock()
|
||||
}
|
||||
|
||||
pub fn interrupt_handle(&self) -> Arc<duckdb::InterruptHandle> {
|
||||
self.interrupt_handle.clone()
|
||||
}
|
||||
|
||||
pub fn mark_draining(&self) {
|
||||
self.draining.store(true, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn clear_draining(&self) {
|
||||
self.draining.store(false, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn is_draining(&self) -> bool {
|
||||
self.draining.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
// Preserve PoisonError ownership so close_connection can still release the contained DuckDB handle.
|
||||
#[allow(clippy::result_large_err)]
|
||||
fn into_inner(self) -> std::sync::LockResult<duckdb::Connection> {
|
||||
self.connection.into_inner()
|
||||
}
|
||||
}
|
||||
|
||||
/// Connects to a DuckDb database file with file validation.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `path` - The file path to the DuckDb database
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Ok(Arc<DuckDbConnection>)` on successful connection
|
||||
/// * `Err(String)` with descriptive error message if connection fails
|
||||
pub fn connect_path(path: &str) -> Result<Arc<DuckDbConnection>, String> {
|
||||
let is_memory = is_memory_database_path(path);
|
||||
if !is_memory {
|
||||
validate_duckdb_path(path)?;
|
||||
remove_empty_placeholder_file(path)?;
|
||||
}
|
||||
|
||||
let connection = if is_memory { duckdb::Connection::open_in_memory() } else { duckdb::Connection::open(path) }
|
||||
.map_err(|e| format!("DuckDb connection failed: {e}"))?;
|
||||
|
||||
Ok(Arc::new(DuckDbConnection::new(connection)))
|
||||
}
|
||||
|
||||
/// Runs a connection init script statement-by-statement so a failure can
|
||||
/// point at the offending statement instead of the whole batch.
|
||||
pub fn run_init_script(con: &duckdb::Connection, script: &str) -> Result<(), String> {
|
||||
for (index, statement) in crate::db::duckdb_sql::split_sql_statements(script).iter().enumerate() {
|
||||
if crate::db::duckdb_sql::strip_leading_comments(statement).is_empty() {
|
||||
continue;
|
||||
}
|
||||
con.execute_batch(statement)
|
||||
.map_err(|e| format!("Connection init script statement {} failed: {e}", index + 1))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_duckdb_path(path: &str) -> Result<(), String> {
|
||||
if path.is_empty() {
|
||||
return Err("Database file path cannot be empty".to_string());
|
||||
}
|
||||
if path.contains('\0') {
|
||||
return Err("Database file path contains null characters".to_string());
|
||||
}
|
||||
if is_network_path(path) {
|
||||
return Ok(());
|
||||
}
|
||||
let path_obj = Path::new(path);
|
||||
if path_obj.is_dir() {
|
||||
return Err(format!("Database file path is a directory, not a file: {}", path));
|
||||
}
|
||||
if !path_obj.exists() {
|
||||
// DuckDB creates a new database file on open when the parent directory
|
||||
// exists. Only reject paths whose parent directory is missing.
|
||||
if let Some(parent) = path_obj.parent() {
|
||||
if !parent.as_os_str().is_empty() && !parent.exists() {
|
||||
return Err(format!("Parent directory does not exist: {}", parent.display()));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn remove_empty_placeholder_file(path: &str) -> Result<(), String> {
|
||||
let path_obj = Path::new(path);
|
||||
if is_network_path(path) || !path_obj.exists() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let metadata = fs::metadata(path_obj).map_err(|e| format!("Failed to read database file metadata: {e}"))?;
|
||||
if metadata.is_file() && metadata.len() == 0 {
|
||||
fs::remove_file(path_obj).map_err(|e| format!("Failed to replace empty DuckDB placeholder file: {e}"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn is_network_path(path: &str) -> bool {
|
||||
path.starts_with("\\\\") || path.starts_with("//") || path.contains("wsl.localhost") || path.contains("wsl$")
|
||||
}
|
||||
|
||||
pub fn is_memory_database_path(path: &str) -> bool {
|
||||
path.trim().eq_ignore_ascii_case(":memory:")
|
||||
}
|
||||
|
||||
/// Closes a DuckDB connection, releasing the file lock.
|
||||
///
|
||||
/// Unlike relying on Drop, this calls `duckdb_disconnect` synchronously
|
||||
/// so the file handle is released before this function returns.
|
||||
/// On Windows this prevents "file already in use" errors when reconnecting.
|
||||
pub fn close_connection(con: Arc<DuckDbConnection>) {
|
||||
match Arc::try_unwrap(con) {
|
||||
Ok(handle) => match handle.into_inner() {
|
||||
Ok(conn) => {
|
||||
let _ = conn.close();
|
||||
}
|
||||
Err(poisoned) => {
|
||||
let _ = poisoned.into_inner().close();
|
||||
}
|
||||
},
|
||||
Err(_) => {
|
||||
// Arc still referenced elsewhere (e.g. running queries);
|
||||
// the last holder will drop and close the connection.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn connect_path_supports_memory_database() {
|
||||
let con = connect_path(":memory:").expect("connect in-memory DuckDB");
|
||||
let locked = con.lock().expect("lock connection");
|
||||
|
||||
locked.execute_batch("CREATE TABLE memory_probe AS SELECT 42 AS id;").expect("create table");
|
||||
let value: i32 = locked.query_row("SELECT id FROM memory_probe;", [], |row| row.get(0)).expect("select row");
|
||||
|
||||
assert_eq!(value, 42);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interrupt_handle_is_available_while_connection_is_locked() {
|
||||
let con = connect_path(":memory:").expect("connect in-memory DuckDB");
|
||||
let _locked = con.lock().expect("lock connection");
|
||||
|
||||
let _interrupt_handle = con.interrupt_handle();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn connect_path_replaces_empty_placeholder_file() {
|
||||
let path = std::env::temp_dir().join(format!("dbx-duckdb-empty-{}.duckdb", uuid::Uuid::new_v4()));
|
||||
std::fs::write(&path, "").expect("write empty placeholder");
|
||||
assert_eq!(std::fs::metadata(&path).expect("placeholder metadata").len(), 0);
|
||||
|
||||
let con = connect_path(path.to_string_lossy().as_ref()).expect("connect empty placeholder DuckDB file");
|
||||
close_connection(con);
|
||||
|
||||
assert!(std::fs::metadata(&path).expect("database metadata").len() > 0);
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn run_init_script_executes_duckdb_escape_strings() {
|
||||
let con = connect_path(":memory:").expect("connect in-memory DuckDB");
|
||||
let locked = con.lock().expect("lock connection");
|
||||
|
||||
run_init_script(&locked, r#"CREATE TABLE probe AS SELECT E'it\'s;ok' AS value; SELECT 2;"#)
|
||||
.expect("run init script");
|
||||
let value: String = locked.query_row("SELECT value FROM probe", [], |row| row.get(0)).expect("select value");
|
||||
|
||||
assert_eq!(value, "it's;ok");
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
//! Quote-aware helpers for inspecting DuckDB SQL snippets without a full
|
||||
//! parser. Shared by the DuckDB worker (execute-time ATTACH tracking) and the
|
||||
//! connection layer (init-script handling), and kept free of `duckdb-bundled`
|
||||
//! so config-derived metadata works in every build.
|
||||
//! connection layer (init-script handling). It stays independent of the native
|
||||
//! DuckDB engine so config-derived metadata remains in the main application.
|
||||
|
||||
use crate::models::connection::ConnectionConfig;
|
||||
|
||||
@@ -130,7 +130,7 @@ pub fn attached_name_from_attach_sql(sql: &str) -> Option<String> {
|
||||
parse_identifier_after_as(&trimmed[as_index + 2..])
|
||||
}
|
||||
|
||||
pub(crate) fn strip_leading_comments(sql: &str) -> &str {
|
||||
pub fn strip_leading_comments(sql: &str) -> &str {
|
||||
let mut rest = sql.trim_start();
|
||||
loop {
|
||||
if let Some(after) = rest.strip_prefix("--") {
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
#![cfg(feature = "duckdb-bundled")]
|
||||
#![cfg(feature = "duckdb-sidecar")]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::ffi::OsString;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Stdio;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex as StdMutex, OnceLock};
|
||||
@@ -14,20 +15,21 @@ use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::db;
|
||||
use crate::db::duckdb_worker_protocol::{
|
||||
DuckDbWorkerConnectParams, DuckDbWorkerError, DuckDbWorkerExecuteParams, DuckDbWorkerMethod,
|
||||
DuckDbWorkerObjectSourceParams, DuckDbWorkerRequest, DuckDbWorkerResponse,
|
||||
DuckDbWorkerColumnParams, DuckDbWorkerConnectParams, DuckDbWorkerError, DuckDbWorkerExecuteParams,
|
||||
DuckDbWorkerMethod, DuckDbWorkerObjectSourceParams, DuckDbWorkerRequest, DuckDbWorkerResponse,
|
||||
};
|
||||
use crate::models::connection::AttachedDatabaseConfig;
|
||||
use crate::storage::{normalize_duckdb_worker_max_processes, DUCKDB_WORKER_MAX_PROCESSES_DEFAULT};
|
||||
|
||||
/// Error code the worker reports when a query error left the DuckDB connection poisoned
|
||||
/// (see the duckdb-rs Parser Error bug). The client kills the worker on this code so the
|
||||
/// next request starts a fresh one. Must match the code emitted in `duckdb_worker_runtime`.
|
||||
/// next request starts a fresh one. Must match the code emitted by the standalone driver runtime.
|
||||
const DUCKDB_WORKER_POISONED_CODE: &str = "duckdb_worker_poisoned";
|
||||
const DUCKDB_WORKER_REQUEST_TIMEOUT_CODE: &str = "duckdb_worker_request_timeout";
|
||||
const DEFAULT_WORKER_REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
const DEFAULT_WORKER_KILL_WAIT: Duration = Duration::from_secs(3);
|
||||
const DEFAULT_WORKER_START_WAIT: Duration = Duration::from_secs(5);
|
||||
pub const DUCKDB_DRIVER_PATH_ENV: &str = "DBX_DUCKDB_DRIVER_PATH";
|
||||
type PendingRequests = Arc<Mutex<HashMap<String, PendingRequest>>>;
|
||||
|
||||
struct PendingRequest {
|
||||
@@ -48,6 +50,7 @@ struct DuckDbWorkerClientInner {
|
||||
process_limiter: Arc<Semaphore>,
|
||||
process_limit: usize,
|
||||
executable: PathBuf,
|
||||
executable_args: Vec<OsString>,
|
||||
connect_params: DuckDbWorkerConnectParams,
|
||||
request_timeout: Duration,
|
||||
worker_start_timeout: Duration,
|
||||
@@ -68,8 +71,16 @@ impl DuckDbWorkerClient {
|
||||
attached_databases: Vec<AttachedDatabaseConfig>,
|
||||
init_script: Option<String>,
|
||||
) -> Result<Self, String> {
|
||||
let executable = std::env::current_exe().map_err(|e| e.to_string())?;
|
||||
Self::open_with_executable(executable, path, attached_databases, init_script).await
|
||||
let (executable, executable_args) = resolve_duckdb_driver_command()?;
|
||||
Self::open_with_command_and_process_limit(
|
||||
executable,
|
||||
executable_args,
|
||||
path,
|
||||
attached_databases,
|
||||
init_script,
|
||||
DUCKDB_WORKER_MAX_PROCESSES_DEFAULT,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn open_with_process_limit(
|
||||
@@ -78,9 +89,16 @@ impl DuckDbWorkerClient {
|
||||
init_script: Option<String>,
|
||||
process_limit: usize,
|
||||
) -> Result<Self, String> {
|
||||
let executable = std::env::current_exe().map_err(|e| e.to_string())?;
|
||||
Self::open_with_executable_and_process_limit(executable, path, attached_databases, init_script, process_limit)
|
||||
.await
|
||||
let (executable, executable_args) = resolve_duckdb_driver_command()?;
|
||||
Self::open_with_command_and_process_limit(
|
||||
executable,
|
||||
executable_args,
|
||||
path,
|
||||
attached_databases,
|
||||
init_script,
|
||||
process_limit,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn open_with_executable(
|
||||
@@ -105,9 +123,29 @@ impl DuckDbWorkerClient {
|
||||
attached_databases: Vec<AttachedDatabaseConfig>,
|
||||
init_script: Option<String>,
|
||||
process_limit: usize,
|
||||
) -> Result<Self, String> {
|
||||
Self::open_with_command_and_process_limit(
|
||||
executable,
|
||||
Vec::new(),
|
||||
path,
|
||||
attached_databases,
|
||||
init_script,
|
||||
process_limit,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn open_with_command_and_process_limit(
|
||||
executable: PathBuf,
|
||||
executable_args: Vec<OsString>,
|
||||
path: String,
|
||||
attached_databases: Vec<AttachedDatabaseConfig>,
|
||||
init_script: Option<String>,
|
||||
process_limit: usize,
|
||||
) -> Result<Self, String> {
|
||||
let client = Self::new_unconnected_with_timeouts(
|
||||
executable,
|
||||
executable_args,
|
||||
path,
|
||||
attached_databases,
|
||||
init_script,
|
||||
@@ -122,6 +160,7 @@ impl DuckDbWorkerClient {
|
||||
#[doc(hidden)]
|
||||
pub fn new_unconnected_with_timeouts(
|
||||
executable: PathBuf,
|
||||
executable_args: Vec<OsString>,
|
||||
path: String,
|
||||
attached_databases: Vec<AttachedDatabaseConfig>,
|
||||
init_script: Option<String>,
|
||||
@@ -139,6 +178,7 @@ impl DuckDbWorkerClient {
|
||||
process_limiter,
|
||||
process_limit,
|
||||
executable,
|
||||
executable_args,
|
||||
connect_params: DuckDbWorkerConnectParams { path, attached_databases, init_script },
|
||||
request_timeout,
|
||||
worker_start_timeout,
|
||||
@@ -246,6 +286,18 @@ impl DuckDbWorkerClient {
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn get_table_ddl(&self, database: String, schema: String, table: String) -> Result<String, String> {
|
||||
self.metadata_request(DuckDbWorkerMethod::GetTableDdl, DuckDbWorkerColumnParams { database, schema, table })
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn completion_assistant(
|
||||
&self,
|
||||
request: db::CompletionAssistantRequest,
|
||||
) -> Result<db::CompletionAssistantResponse, String> {
|
||||
self.metadata_request(DuckDbWorkerMethod::CompletionAssistant, request).await
|
||||
}
|
||||
|
||||
pub async fn get_object_source(
|
||||
&self,
|
||||
database: String,
|
||||
@@ -413,9 +465,14 @@ impl DuckDbWorkerClient {
|
||||
.map_err(|_| duckdb_worker_process_limit_error(self.inner.process_limit))?
|
||||
.map_err(|_| "DuckDB worker process limiter is closed".to_string())?;
|
||||
|
||||
log::info!("[duckdb-worker:start] executable={}", self.inner.executable.display());
|
||||
let mut child = crate::process::new_tokio_command(&self.inner.executable)
|
||||
.arg("--duckdb-worker")
|
||||
log::info!(
|
||||
"[duckdb-worker:start] executable={} args={:?}",
|
||||
self.inner.executable.display(),
|
||||
self.inner.executable_args
|
||||
);
|
||||
let mut command = crate::process::new_tokio_command(&self.inner.executable);
|
||||
command.args(&self.inner.executable_args);
|
||||
let mut child = command
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::inherit())
|
||||
@@ -524,6 +581,25 @@ impl DuckDbWorkerClient {
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_duckdb_driver_command() -> Result<(PathBuf, Vec<OsString>), String> {
|
||||
if let Some(executable) = std::env::var_os(DUCKDB_DRIVER_PATH_ENV).filter(|value| !value.is_empty()) {
|
||||
let executable = PathBuf::from(executable);
|
||||
ensure_duckdb_driver_exists(&executable)?;
|
||||
return Ok((executable, Vec::new()));
|
||||
}
|
||||
|
||||
Err(format!(
|
||||
"DuckDB driver is not installed. Please install it from the Driver Manager or set {DUCKDB_DRIVER_PATH_ENV}."
|
||||
))
|
||||
}
|
||||
|
||||
fn ensure_duckdb_driver_exists(executable: &Path) -> Result<(), String> {
|
||||
if executable.is_file() {
|
||||
return Ok(());
|
||||
}
|
||||
Err(format!("DuckDB driver configured by {DUCKDB_DRIVER_PATH_ENV} was not found: {}", executable.display()))
|
||||
}
|
||||
|
||||
struct WorkerProcessLimiterState {
|
||||
limit: usize,
|
||||
limiter: Arc<Semaphore>,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#![cfg(feature = "duckdb-bundled")]
|
||||
#![cfg(feature = "duckdb-sidecar")]
|
||||
|
||||
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
||||
|
||||
@@ -35,6 +35,8 @@ pub enum DuckDbWorkerMethod {
|
||||
ListSchemas,
|
||||
ListTables,
|
||||
ListColumns,
|
||||
GetTableDdl,
|
||||
CompletionAssistant,
|
||||
GetObjectSource,
|
||||
AttachDatabase,
|
||||
Cancel,
|
||||
|
||||
@@ -1,642 +0,0 @@
|
||||
#![cfg(feature = "duckdb-bundled")]
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Stdout};
|
||||
|
||||
use crate::db;
|
||||
use crate::db::duckdb_worker_protocol::{
|
||||
DuckDbWorkerColumnParams, DuckDbWorkerConnectParams, DuckDbWorkerDatabaseParams, DuckDbWorkerError,
|
||||
DuckDbWorkerExecuteParams, DuckDbWorkerMethod, DuckDbWorkerObjectSourceParams, DuckDbWorkerRequest,
|
||||
DuckDbWorkerResponse, DuckDbWorkerTableParams,
|
||||
};
|
||||
use crate::models::connection::AttachedDatabaseConfig;
|
||||
use crate::path_utils::expand_tilde;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct DuckDbWorkerSession {
|
||||
connection: Option<Arc<db::duckdb_driver::DuckDbConnection>>,
|
||||
attached_names: Vec<String>,
|
||||
}
|
||||
|
||||
impl DuckDbWorkerSession {
|
||||
pub fn connect(&mut self, params: DuckDbWorkerConnectParams) -> Result<(), String> {
|
||||
let path = expand_tilde(¶ms.path);
|
||||
let connection = db::duckdb_driver::connect_path(&path)?;
|
||||
let mut attached_names = Vec::new();
|
||||
{
|
||||
let locked = connection.lock().map_err(|e| e.to_string())?;
|
||||
for attached in ¶ms.attached_databases {
|
||||
let path = expand_tilde(&attached.path);
|
||||
crate::schema::duckdb_attach_database(&locked, &attached.name, &path)?;
|
||||
attached_names.push(attached.name.clone());
|
||||
}
|
||||
if let Some(script) = params.init_script.as_deref() {
|
||||
// Track script attachments by diffing the actual catalog list
|
||||
// instead of parsing the SQL, so ATTACH without an alias,
|
||||
// dynamic statements, and extension-specific syntax all count.
|
||||
let before = duckdb_catalog_list(&locked)?;
|
||||
db::duckdb_driver::run_init_script(&locked, script)?;
|
||||
for name in duckdb_catalog_list(&locked)? {
|
||||
if !before.iter().any(|existing| existing.eq_ignore_ascii_case(&name))
|
||||
&& !attached_names.iter().any(|attached| attached.eq_ignore_ascii_case(&name))
|
||||
{
|
||||
attached_names.push(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
self.connection = Some(connection);
|
||||
self.attached_names = attached_names;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn execute(&mut self, params: DuckDbWorkerExecuteParams) -> Result<db::QueryResult, String> {
|
||||
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
|
||||
let locked = connection.lock().map_err(|e| e.to_string())?;
|
||||
let result = crate::query::duckdb_execute_for_database(
|
||||
&locked,
|
||||
&self.attached_names,
|
||||
params.database.as_deref(),
|
||||
¶ms.sql,
|
||||
params.max_rows,
|
||||
)?;
|
||||
if let Some(name) = crate::db::duckdb_sql::attached_name_from_attach_sql(¶ms.sql) {
|
||||
if !self.attached_names.iter().any(|attached| attached.eq_ignore_ascii_case(&name)) {
|
||||
self.attached_names.push(name);
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn list_databases(&self) -> Result<Vec<db::DatabaseInfo>, String> {
|
||||
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
|
||||
let locked = connection.lock().map_err(|e| e.to_string())?;
|
||||
crate::schema::duckdb_list_databases_with_attached(&locked, &self.attached_names)
|
||||
}
|
||||
|
||||
pub fn list_schemas(&self, params: DuckDbWorkerDatabaseParams) -> Result<Vec<String>, String> {
|
||||
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
|
||||
let locked = connection.lock().map_err(|e| e.to_string())?;
|
||||
crate::schema::duckdb_list_schemas_with_attached(&locked, ¶ms.database, &self.attached_names)
|
||||
}
|
||||
|
||||
pub fn list_tables(&self, params: DuckDbWorkerTableParams) -> Result<Vec<db::TableInfo>, String> {
|
||||
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
|
||||
let locked = connection.lock().map_err(|e| e.to_string())?;
|
||||
crate::schema::duckdb_query_tables_in_database_with_attached(
|
||||
&locked,
|
||||
¶ms.database,
|
||||
¶ms.schema,
|
||||
&self.attached_names,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn list_columns(&self, params: DuckDbWorkerColumnParams) -> Result<Vec<db::ColumnInfo>, String> {
|
||||
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
|
||||
let locked = connection.lock().map_err(|e| e.to_string())?;
|
||||
crate::schema::duckdb_query_columns_in_database_with_attached(
|
||||
&locked,
|
||||
¶ms.database,
|
||||
¶ms.schema,
|
||||
¶ms.table,
|
||||
&self.attached_names,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn get_object_source(&self, params: DuckDbWorkerObjectSourceParams) -> Result<String, String> {
|
||||
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
|
||||
let locked = connection.lock().map_err(|e| e.to_string())?;
|
||||
crate::schema::duckdb_object_source_with_attached(
|
||||
&locked,
|
||||
¶ms.database,
|
||||
¶ms.schema,
|
||||
¶ms.name,
|
||||
¶ms.object_type,
|
||||
&self.attached_names,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn attach_database(&mut self, attached: AttachedDatabaseConfig) -> Result<(), String> {
|
||||
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
|
||||
let locked = connection.lock().map_err(|e| e.to_string())?;
|
||||
let path = expand_tilde(&attached.path);
|
||||
crate::schema::duckdb_attach_database(&locked, &attached.name, &path)?;
|
||||
self.attached_names.push(attached.name);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn interrupt_handle(&self) -> Result<Arc<duckdb::InterruptHandle>, String> {
|
||||
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
|
||||
Ok(connection.interrupt_handle())
|
||||
}
|
||||
|
||||
/// Probes whether the connection is still usable after an execute error.
|
||||
///
|
||||
/// duckdb-rs 1.10503.1 has a bug where `Connection::prepare()` failing at the
|
||||
/// `duckdb_extract_statements` stage (a syntax `Parser Error`) permanently poisons
|
||||
/// the connection: every later operation returns `resource deadlock would occur`,
|
||||
/// and dropping the connection aborts the whole process. Benign errors (binder,
|
||||
/// catalog, runtime) do not poison the connection.
|
||||
///
|
||||
/// This probe runs a trivial query through `execute_batch`, which uses the
|
||||
/// non-poisoning `duckdb_query_arrow` path and fails fast on a poisoned connection.
|
||||
/// It returns `true` when the connection is healthy and safe to reuse.
|
||||
fn is_connection_healthy(&self) -> bool {
|
||||
match self.connection.as_ref() {
|
||||
Some(connection) => match connection.lock() {
|
||||
Ok(locked) => locked.execute_batch("SELECT 1").is_ok(),
|
||||
Err(_) => false,
|
||||
},
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn duckdb_catalog_list(con: &duckdb::Connection) -> Result<Vec<String>, String> {
|
||||
let mut stmt = con.prepare("SHOW DATABASES").map_err(|e| e.to_string())?;
|
||||
let rows = stmt.query_map([], |row| row.get::<_, String>(0)).map_err(|e| e.to_string())?;
|
||||
Ok(rows.filter_map(|row| row.ok()).collect())
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct DuckDbWorkerRuntime {
|
||||
session: Arc<Mutex<DuckDbWorkerSession>>,
|
||||
active_interrupt: Arc<Mutex<Option<Arc<duckdb::InterruptHandle>>>>,
|
||||
}
|
||||
|
||||
struct WorkerHandleResult {
|
||||
response: Option<DuckDbWorkerResponse>,
|
||||
shutdown: bool,
|
||||
}
|
||||
|
||||
impl DuckDbWorkerRuntime {
|
||||
async fn handle_request(
|
||||
&self,
|
||||
request: DuckDbWorkerRequest,
|
||||
stdout: Arc<tokio::sync::Mutex<Stdout>>,
|
||||
) -> WorkerHandleResult {
|
||||
match request.method {
|
||||
DuckDbWorkerMethod::Connect => self.handle_session_request(request, |session, request| {
|
||||
session.connect(request.parse_params()?)?;
|
||||
Ok(DuckDbWorkerResponse::ok(request.id, serde_json::json!({ "connected": true })))
|
||||
}),
|
||||
DuckDbWorkerMethod::ListDatabases => self.handle_session_request(request, |session, request| {
|
||||
Ok(DuckDbWorkerResponse::ok(request.id, session.list_databases()?))
|
||||
}),
|
||||
DuckDbWorkerMethod::ListSchemas => self.handle_session_request(request, |session, request| {
|
||||
let params = request.parse_params()?;
|
||||
Ok(DuckDbWorkerResponse::ok(request.id, session.list_schemas(params)?))
|
||||
}),
|
||||
DuckDbWorkerMethod::ListTables => self.handle_session_request(request, |session, request| {
|
||||
let params = request.parse_params()?;
|
||||
Ok(DuckDbWorkerResponse::ok(request.id, session.list_tables(params)?))
|
||||
}),
|
||||
DuckDbWorkerMethod::ListColumns => self.handle_session_request(request, |session, request| {
|
||||
let params = request.parse_params()?;
|
||||
Ok(DuckDbWorkerResponse::ok(request.id, session.list_columns(params)?))
|
||||
}),
|
||||
DuckDbWorkerMethod::GetObjectSource => self.handle_session_request(request, |session, request| {
|
||||
let params = request.parse_params()?;
|
||||
Ok(DuckDbWorkerResponse::ok(request.id, session.get_object_source(params)?))
|
||||
}),
|
||||
DuckDbWorkerMethod::AttachDatabase => self.handle_session_request(request, |session, request| {
|
||||
session.attach_database(request.parse_params()?)?;
|
||||
Ok(DuckDbWorkerResponse::ok_empty(request.id))
|
||||
}),
|
||||
DuckDbWorkerMethod::Execute => {
|
||||
if self.active_interrupt.lock().unwrap_or_else(|e| e.into_inner()).is_some() {
|
||||
return WorkerHandleResult {
|
||||
response: Some(DuckDbWorkerResponse::err(
|
||||
request.id,
|
||||
DuckDbWorkerError::new("duckdb_worker_busy", "DuckDB worker is already executing a query"),
|
||||
)),
|
||||
shutdown: false,
|
||||
};
|
||||
}
|
||||
|
||||
let interrupt_handle = match self.session.lock().unwrap_or_else(|e| e.into_inner()).interrupt_handle() {
|
||||
Ok(handle) => handle,
|
||||
Err(err) => {
|
||||
return WorkerHandleResult {
|
||||
response: Some(DuckDbWorkerResponse::err(
|
||||
request.id,
|
||||
DuckDbWorkerError::from_message("duckdb_not_connected", err),
|
||||
)),
|
||||
shutdown: false,
|
||||
}
|
||||
}
|
||||
};
|
||||
*self.active_interrupt.lock().unwrap_or_else(|e| e.into_inner()) = Some(interrupt_handle);
|
||||
|
||||
let session = self.session.clone();
|
||||
let active_interrupt = self.active_interrupt.clone();
|
||||
tokio::spawn(async move {
|
||||
let id = request.id.clone();
|
||||
let params = request.parse_params::<DuckDbWorkerExecuteParams>();
|
||||
let (result, poisoned) = match params {
|
||||
Ok(params) => {
|
||||
let probe_session = session.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let mut session = probe_session.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let result = session.execute(params);
|
||||
// Only probe connection health when execute failed: a syntax
|
||||
// Parser Error can poison the connection (duckdb-rs bug), and any
|
||||
// later use — including drop — would abort the process.
|
||||
let poisoned = result.is_err() && !session.is_connection_healthy();
|
||||
(result, poisoned)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
.unwrap_or_else(|err| (Err(err), true))
|
||||
}
|
||||
Err(err) => (Err(err.message), false),
|
||||
};
|
||||
*active_interrupt.lock().unwrap_or_else(|e| e.into_inner()) = None;
|
||||
let response = match result {
|
||||
Ok(result) => DuckDbWorkerResponse::ok(id, result),
|
||||
// A poisoned connection cannot be reused or even safely dropped in-process.
|
||||
// Report a distinct code so the parent kills this worker (OS-level, skipping
|
||||
// destructors) and restarts a fresh one on the next request. We must not exit
|
||||
// here ourselves: process::exit races the parent's next request, which could
|
||||
// be written to a dying worker. Killing on the parent side has no such window.
|
||||
Err(err) if poisoned => {
|
||||
log::warn!("[duckdb-worker:poisoned-connection] reporting to parent for restart");
|
||||
DuckDbWorkerResponse::err(
|
||||
id,
|
||||
DuckDbWorkerError::from_message("duckdb_worker_poisoned", err),
|
||||
)
|
||||
}
|
||||
Err(err) => {
|
||||
DuckDbWorkerResponse::err(id, DuckDbWorkerError::from_message("duckdb_execute_failed", err))
|
||||
}
|
||||
};
|
||||
write_response(stdout, &response).await;
|
||||
});
|
||||
|
||||
WorkerHandleResult { response: None, shutdown: false }
|
||||
}
|
||||
DuckDbWorkerMethod::Cancel => {
|
||||
if let Some(handle) = self.active_interrupt.lock().unwrap_or_else(|e| e.into_inner()).as_ref() {
|
||||
handle.interrupt();
|
||||
}
|
||||
WorkerHandleResult {
|
||||
response: Some(DuckDbWorkerResponse::ok(request.id, serde_json::json!({ "cancelled": true }))),
|
||||
shutdown: false,
|
||||
}
|
||||
}
|
||||
DuckDbWorkerMethod::Shutdown => WorkerHandleResult {
|
||||
response: Some(DuckDbWorkerResponse::ok(request.id, serde_json::json!({ "shutdown": true }))),
|
||||
shutdown: true,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_session_request(
|
||||
&self,
|
||||
request: DuckDbWorkerRequest,
|
||||
handler: impl FnOnce(
|
||||
&mut DuckDbWorkerSession,
|
||||
DuckDbWorkerRequest,
|
||||
) -> Result<DuckDbWorkerResponse, DuckDbWorkerError>,
|
||||
) -> WorkerHandleResult {
|
||||
let id = request.id.clone();
|
||||
if self.active_interrupt.lock().unwrap_or_else(|e| e.into_inner()).is_some() {
|
||||
return WorkerHandleResult {
|
||||
response: Some(DuckDbWorkerResponse::err(
|
||||
id,
|
||||
DuckDbWorkerError::new("duckdb_worker_busy", "DuckDB worker is executing a query"),
|
||||
)),
|
||||
shutdown: false,
|
||||
};
|
||||
}
|
||||
let response = {
|
||||
match self.session.try_lock() {
|
||||
Ok(mut session) => match handler(&mut session, request) {
|
||||
Ok(response) => response,
|
||||
Err(err) => DuckDbWorkerResponse::err(id, err),
|
||||
},
|
||||
Err(_) => DuckDbWorkerResponse::err(
|
||||
id,
|
||||
DuckDbWorkerError::new("duckdb_worker_busy", "DuckDB worker session is busy"),
|
||||
),
|
||||
}
|
||||
};
|
||||
WorkerHandleResult { response: Some(response), shutdown: false }
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run_stdio_worker() -> Result<(), String> {
|
||||
let runtime = DuckDbWorkerRuntime::default();
|
||||
let stdout = Arc::new(tokio::sync::Mutex::new(tokio::io::stdout()));
|
||||
let stdin = BufReader::new(tokio::io::stdin());
|
||||
let mut lines = stdin.lines();
|
||||
|
||||
loop {
|
||||
let line = lines.next_line().await.map_err(|e| e.to_string())?;
|
||||
let Some(line) = line else {
|
||||
std::process::exit(0);
|
||||
};
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let request: DuckDbWorkerRequest = match serde_json::from_str(&line) {
|
||||
Ok(request) => request,
|
||||
Err(err) => {
|
||||
let response = DuckDbWorkerResponse::err(
|
||||
"",
|
||||
DuckDbWorkerError::new("invalid_json", format!("Invalid worker request JSON: {err}")),
|
||||
);
|
||||
write_response(stdout.clone(), &response).await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let result = runtime.handle_request(request, stdout.clone()).await;
|
||||
if let Some(response) = result.response {
|
||||
write_response(stdout.clone(), &response).await;
|
||||
}
|
||||
if result.shutdown {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn write_response(stdout: Arc<tokio::sync::Mutex<Stdout>>, response: &DuckDbWorkerResponse) {
|
||||
let Ok(line) = serde_json::to_string(response) else {
|
||||
return;
|
||||
};
|
||||
let mut stdout = stdout.lock().await;
|
||||
let _ = stdout.write_all(line.as_bytes()).await;
|
||||
let _ = stdout.write_all(b"\n").await;
|
||||
let _ = stdout.flush().await;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn worker_session_connects_to_memory_duckdb() {
|
||||
let mut session = DuckDbWorkerSession::default();
|
||||
|
||||
session
|
||||
.connect(DuckDbWorkerConnectParams {
|
||||
path: ":memory:".to_string(),
|
||||
attached_databases: Vec::new(),
|
||||
init_script: None,
|
||||
})
|
||||
.expect("connect");
|
||||
|
||||
assert_eq!(session.list_databases().expect("list databases")[0].name, "main");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_session_executes_select_query() {
|
||||
let mut session = DuckDbWorkerSession::default();
|
||||
session
|
||||
.connect(DuckDbWorkerConnectParams {
|
||||
path: ":memory:".to_string(),
|
||||
attached_databases: Vec::new(),
|
||||
init_script: None,
|
||||
})
|
||||
.expect("connect");
|
||||
|
||||
let result = session
|
||||
.execute(DuckDbWorkerExecuteParams { sql: "SELECT 1 AS value".to_string(), database: None, max_rows: None })
|
||||
.expect("execute");
|
||||
|
||||
assert_eq!(result.columns, vec!["value"]);
|
||||
assert_eq!(result.rows, vec![vec![serde_json::json!(1)]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_session_lists_tables_and_columns() {
|
||||
let mut session = DuckDbWorkerSession::default();
|
||||
session
|
||||
.connect(DuckDbWorkerConnectParams {
|
||||
path: ":memory:".to_string(),
|
||||
attached_databases: Vec::new(),
|
||||
init_script: None,
|
||||
})
|
||||
.expect("connect");
|
||||
session
|
||||
.execute(DuckDbWorkerExecuteParams {
|
||||
sql: "CREATE TABLE users (id INTEGER PRIMARY KEY, name VARCHAR)".to_string(),
|
||||
database: None,
|
||||
max_rows: None,
|
||||
})
|
||||
.expect("create table");
|
||||
|
||||
let schemas =
|
||||
session.list_schemas(DuckDbWorkerDatabaseParams { database: "main".to_string() }).expect("list schemas");
|
||||
let tables = session
|
||||
.list_tables(DuckDbWorkerTableParams { database: "main".to_string(), schema: "main".to_string() })
|
||||
.expect("list tables");
|
||||
let columns = session
|
||||
.list_columns(DuckDbWorkerColumnParams {
|
||||
database: "main".to_string(),
|
||||
schema: "main".to_string(),
|
||||
table: "users".to_string(),
|
||||
})
|
||||
.expect("list columns");
|
||||
|
||||
assert!(schemas.iter().any(|schema| schema == "main"));
|
||||
assert!(tables.iter().any(|table| table.name == "users"));
|
||||
assert!(columns.iter().any(|column| column.name == "id" && column.is_primary_key));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_session_reads_view_source() {
|
||||
let mut session = DuckDbWorkerSession::default();
|
||||
session
|
||||
.connect(DuckDbWorkerConnectParams {
|
||||
path: ":memory:".to_string(),
|
||||
attached_databases: Vec::new(),
|
||||
init_script: None,
|
||||
})
|
||||
.expect("connect");
|
||||
session
|
||||
.execute(DuckDbWorkerExecuteParams {
|
||||
sql: "CREATE VIEW active_orders AS SELECT 1 AS id".to_string(),
|
||||
database: None,
|
||||
max_rows: None,
|
||||
})
|
||||
.expect("create view");
|
||||
|
||||
let source = session
|
||||
.get_object_source(DuckDbWorkerObjectSourceParams {
|
||||
database: "main".to_string(),
|
||||
schema: "main".to_string(),
|
||||
name: "active_orders".to_string(),
|
||||
object_type: db::ObjectSourceKind::View,
|
||||
})
|
||||
.expect("get view source");
|
||||
|
||||
assert!(source.starts_with("CREATE VIEW"));
|
||||
assert!(source.contains("active_orders"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_session_attaches_database() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-duckdb-worker-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).expect("create temp dir");
|
||||
let attached_path = dir.join("analytics.duckdb");
|
||||
{
|
||||
let con = duckdb::Connection::open(&attached_path).expect("create attached db");
|
||||
con.execute_batch("CREATE TABLE events (id INTEGER);").expect("create attached table");
|
||||
}
|
||||
let mut session = DuckDbWorkerSession::default();
|
||||
session
|
||||
.connect(DuckDbWorkerConnectParams {
|
||||
path: ":memory:".to_string(),
|
||||
attached_databases: Vec::new(),
|
||||
init_script: None,
|
||||
})
|
||||
.expect("connect");
|
||||
|
||||
session
|
||||
.attach_database(AttachedDatabaseConfig {
|
||||
name: "analytics".to_string(),
|
||||
path: attached_path.to_string_lossy().to_string(),
|
||||
})
|
||||
.expect("attach");
|
||||
|
||||
let databases = session.list_databases().expect("list databases");
|
||||
let tables = session
|
||||
.list_tables(DuckDbWorkerTableParams { database: "analytics".to_string(), schema: "main".to_string() })
|
||||
.expect("list tables");
|
||||
assert!(databases.iter().any(|database| database.name == "analytics"));
|
||||
assert!(tables.iter().any(|table| table.name == "events"));
|
||||
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_session_runs_init_script_and_tracks_attached_alias() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-duckdb-worker-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).expect("create temp dir");
|
||||
let attached_path = dir.join("warehouse.duckdb");
|
||||
{
|
||||
let con = duckdb::Connection::open(&attached_path).expect("create attached db");
|
||||
con.execute_batch("CREATE TABLE facts (id INTEGER);").expect("create attached table");
|
||||
}
|
||||
let mut session = DuckDbWorkerSession::default();
|
||||
let script = format!(
|
||||
"SET memory_limit='512MB'; -- session tuning\nATTACH '{}' AS warehouse;\nCREATE TABLE staging (id INTEGER);",
|
||||
attached_path.to_string_lossy().replace('\'', "''")
|
||||
);
|
||||
session
|
||||
.connect(DuckDbWorkerConnectParams {
|
||||
path: ":memory:".to_string(),
|
||||
attached_databases: Vec::new(),
|
||||
init_script: Some(script),
|
||||
})
|
||||
.expect("connect");
|
||||
|
||||
assert!(session.attached_names.iter().any(|name| name == "warehouse"));
|
||||
let databases = session.list_databases().expect("list databases");
|
||||
assert!(databases.iter().any(|database| database.name == "warehouse"));
|
||||
let tables = session
|
||||
.list_tables(DuckDbWorkerTableParams { database: "warehouse".to_string(), schema: "main".to_string() })
|
||||
.expect("list tables");
|
||||
assert!(tables.iter().any(|table| table.name == "facts"));
|
||||
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_session_tracks_init_script_attach_without_alias() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-duckdb-worker-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).expect("create temp dir");
|
||||
let attached_path = dir.join("noalias.duckdb");
|
||||
{
|
||||
let con = duckdb::Connection::open(&attached_path).expect("create attached db");
|
||||
con.execute_batch("CREATE TABLE items (id INTEGER);").expect("create attached table");
|
||||
}
|
||||
let mut session = DuckDbWorkerSession::default();
|
||||
session
|
||||
.connect(DuckDbWorkerConnectParams {
|
||||
path: ":memory:".to_string(),
|
||||
attached_databases: Vec::new(),
|
||||
init_script: Some(format!("ATTACH '{}';", attached_path.to_string_lossy().replace('\'', "''"))),
|
||||
})
|
||||
.expect("connect");
|
||||
|
||||
assert!(session.attached_names.iter().any(|name| name == "noalias"), "names: {:?}", session.attached_names);
|
||||
let tables = session
|
||||
.list_tables(DuckDbWorkerTableParams { database: "noalias".to_string(), schema: "main".to_string() })
|
||||
.expect("list tables");
|
||||
assert!(tables.iter().any(|table| table.name == "items"));
|
||||
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_session_reports_failing_init_script_statement() {
|
||||
let mut session = DuckDbWorkerSession::default();
|
||||
let err = session
|
||||
.connect(DuckDbWorkerConnectParams {
|
||||
path: ":memory:".to_string(),
|
||||
attached_databases: Vec::new(),
|
||||
init_script: Some("SELECT 1; SELECT * FROM missing_table;".to_string()),
|
||||
})
|
||||
.expect_err("init script should fail");
|
||||
assert!(err.contains("statement 2"), "unexpected error: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_session_tracks_attach_sql_alias() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-duckdb-worker-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).expect("create temp dir");
|
||||
let attached_path = dir.join("sales.duckdb");
|
||||
{
|
||||
let con = duckdb::Connection::open(&attached_path).expect("create attached db");
|
||||
con.execute_batch("CREATE TABLE orders (id INTEGER);").expect("create attached table");
|
||||
}
|
||||
let mut session = DuckDbWorkerSession::default();
|
||||
session
|
||||
.connect(DuckDbWorkerConnectParams {
|
||||
path: ":memory:".to_string(),
|
||||
attached_databases: Vec::new(),
|
||||
init_script: None,
|
||||
})
|
||||
.expect("connect");
|
||||
|
||||
session
|
||||
.execute(DuckDbWorkerExecuteParams {
|
||||
sql: format!("ATTACH '{}' AS \"sales db\";", attached_path.to_string_lossy().replace('\'', "''")),
|
||||
database: None,
|
||||
max_rows: None,
|
||||
})
|
||||
.expect("attach sql");
|
||||
|
||||
assert!(session.attached_names.iter().any(|name| name == "sales db"));
|
||||
let tables = session
|
||||
.list_tables(DuckDbWorkerTableParams { database: "sales db".to_string(), schema: "main".to_string() })
|
||||
.expect("list tables");
|
||||
assert!(tables.iter().any(|table| table.name == "orders"));
|
||||
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_returns_busy_while_query_is_active() {
|
||||
let runtime = DuckDbWorkerRuntime::default();
|
||||
let stdout = Arc::new(tokio::sync::Mutex::new(tokio::io::stdout()));
|
||||
*runtime.active_interrupt.lock().unwrap_or_else(|e| e.into_inner()) =
|
||||
Some(duckdb::Connection::open_in_memory().expect("connection").interrupt_handle());
|
||||
|
||||
let result = runtime
|
||||
.handle_request(
|
||||
DuckDbWorkerRequest::new("req-busy", DuckDbWorkerMethod::ListDatabases, serde_json::json!({}))
|
||||
.expect("request"),
|
||||
stdout,
|
||||
)
|
||||
.await;
|
||||
|
||||
let response = result.response.expect("response");
|
||||
assert!(!response.ok);
|
||||
assert_eq!(response.error.expect("error").code, "duckdb_worker_busy");
|
||||
}
|
||||
}
|
||||
@@ -5,14 +5,11 @@ pub mod cloudflare_d1;
|
||||
pub use cloudflare_d1 as cloudflare_d1_driver;
|
||||
pub mod document_result;
|
||||
pub mod doris;
|
||||
pub mod duckdb_driver;
|
||||
pub mod duckdb_sql;
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
pub mod duckdb_worker_process;
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
pub mod duckdb_worker_protocol;
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
pub mod duckdb_worker_runtime;
|
||||
pub mod elasticsearch_driver;
|
||||
pub mod elasticsearch_sql;
|
||||
pub mod file_validator;
|
||||
|
||||
@@ -63,7 +63,7 @@ pub enum DatabaseCreationTarget {
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
pub struct DuckDbAttachDatabaseSqlOptions {
|
||||
pub path: String,
|
||||
pub name: String,
|
||||
@@ -312,7 +312,7 @@ pub fn supports_database_property_comment(database_type: Option<DatabaseType>) -
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
pub fn build_duckdb_attach_database_sql(options: DuckDbAttachDatabaseSqlOptions) -> String {
|
||||
format!(
|
||||
"ATTACH {} AS {};",
|
||||
@@ -1096,7 +1096,7 @@ mod tests {
|
||||
assert!(!supports_create_database_charset(Some(DatabaseType::Postgres), None));
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
#[test]
|
||||
fn builds_duckdb_attach_sql() {
|
||||
assert_eq!(
|
||||
|
||||
-173
@@ -1,173 +0,0 @@
|
||||
#![cfg(feature = "duckdb-bundled")]
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::types::{CacheState, ExternalTableRef, ExternalTableSnapshot};
|
||||
|
||||
/// Loads an external table snapshot into an in-memory DuckDB connection.
|
||||
pub fn load_snapshot_to_duckdb(con: &duckdb::Connection, snapshot: &ExternalTableSnapshot) -> Result<(), String> {
|
||||
let table_name = sanitize_table_name(&snapshot.table_ref.table_name);
|
||||
|
||||
con.execute(&format!("DROP TABLE IF EXISTS \"{}\"", table_name), [])
|
||||
.map_err(|e| format!("Failed to drop table: {e}"))?;
|
||||
|
||||
let col_defs: Vec<String> = snapshot
|
||||
.columns
|
||||
.iter()
|
||||
.map(|c| {
|
||||
format!(
|
||||
"\"{}\" {}{}",
|
||||
c.name.replace('"', "\"\""),
|
||||
c.duckdb_type,
|
||||
if c.is_nullable { "" } else { " NOT NULL" }
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let pk_cols: Vec<&str> = snapshot.columns.iter().filter(|c| c.is_primary_key).map(|c| c.name.as_str()).collect();
|
||||
|
||||
let mut create_sql = format!("CREATE TABLE \"{}\" ({}", table_name, col_defs.join(", "));
|
||||
if !pk_cols.is_empty() {
|
||||
create_sql.push_str(&format!(
|
||||
", PRIMARY KEY ({})",
|
||||
pk_cols.iter().map(|c| format!("\"{}\"", c.replace('"', "\"\""))).collect::<Vec<_>>().join(", ")
|
||||
));
|
||||
}
|
||||
create_sql.push(')');
|
||||
|
||||
con.execute(&create_sql, []).map_err(|e| format!("Failed to create table: {e}"))?;
|
||||
|
||||
if snapshot.rows.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let placeholders: Vec<&str> = (0..snapshot.columns.len()).map(|_| "?").collect();
|
||||
let insert_sql = format!("INSERT INTO \"{}\" VALUES ({})", table_name, placeholders.join(", "));
|
||||
let mut stmt = con.prepare(&insert_sql).map_err(|e| format!("Failed to prepare insert: {e}"))?;
|
||||
|
||||
for row in &snapshot.rows {
|
||||
let params: Vec<Box<dyn duckdb::ToSql>> = row
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, val)| {
|
||||
let col_type = snapshot.columns.get(i).map(|c| c.duckdb_type.as_str()).unwrap_or("VARCHAR");
|
||||
json_to_duckdb_param(val, col_type)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let param_refs: Vec<&dyn duckdb::ToSql> = params.iter().map(|p| p.as_ref()).collect();
|
||||
stmt.execute(param_refs.as_slice()).map_err(|e| format!("Failed to insert row: {e}"))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn json_to_duckdb_param(val: &serde_json::Value, col_type: &str) -> Box<dyn duckdb::ToSql> {
|
||||
match val {
|
||||
serde_json::Value::Null => Box::new(None::<String>),
|
||||
serde_json::Value::Bool(b) => Box::new(*b),
|
||||
serde_json::Value::Number(n) => {
|
||||
if col_type.contains("INT") {
|
||||
Box::new(n.as_i64().unwrap_or(0))
|
||||
} else if col_type.contains("DOUBLE") || col_type.contains("FLOAT") || col_type.contains("DECIMAL") {
|
||||
Box::new(n.as_f64().unwrap_or(0.0))
|
||||
} else {
|
||||
Box::new(n.to_string())
|
||||
}
|
||||
}
|
||||
serde_json::Value::String(s) => Box::new(s.clone()),
|
||||
_ => Box::new(val.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Sanitize a source table name into a stable DuckDB table identifier.
|
||||
pub fn sanitize_table_name(name: &str) -> String {
|
||||
let sanitized: String = name.chars().map(|c| if c.is_alphanumeric() || c == '_' { c } else { '_' }).collect();
|
||||
|
||||
if sanitized.is_empty() {
|
||||
"_unnamed_".to_string()
|
||||
} else if sanitized.chars().next().is_some_and(|c| c.is_ascii_digit()) {
|
||||
format!("_{sanitized}")
|
||||
} else {
|
||||
sanitized
|
||||
}
|
||||
}
|
||||
|
||||
/// Manages a DuckDB cache for one external source.
|
||||
pub struct ExternalPool {
|
||||
pub source: Arc<dyn super::traits::ExternalTabularSource>,
|
||||
pub cache: Arc<std::sync::Mutex<duckdb::Connection>>,
|
||||
pub cache_state: std::sync::Mutex<CacheState>,
|
||||
pub table_map: std::sync::Mutex<std::collections::HashMap<String, ExternalTableRef>>,
|
||||
}
|
||||
|
||||
impl ExternalPool {
|
||||
pub fn new(
|
||||
source: Arc<dyn super::traits::ExternalTabularSource>,
|
||||
cache: Arc<std::sync::Mutex<duckdb::Connection>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
source,
|
||||
cache,
|
||||
cache_state: std::sync::Mutex::new(CacheState::Empty),
|
||||
table_map: std::sync::Mutex::new(std::collections::HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn refresh_cache(&self) -> Result<(), String> {
|
||||
{
|
||||
let mut state = self.cache_state.lock().map_err(|e| e.to_string())?;
|
||||
*state = CacheState::Loading;
|
||||
}
|
||||
|
||||
let tables = self.source.list_tables().await?;
|
||||
let mut new_table_map = std::collections::HashMap::new();
|
||||
|
||||
for table_ref in &tables {
|
||||
let snapshot = self.source.load_table(table_ref).await?;
|
||||
let mut sanitized_name = sanitize_table_name(&table_ref.table_name);
|
||||
|
||||
if new_table_map.contains_key(&sanitized_name) {
|
||||
let base = sanitized_name.clone();
|
||||
let mut suffix = 2;
|
||||
loop {
|
||||
sanitized_name = format!("{base}_{suffix}");
|
||||
if !new_table_map.contains_key(&sanitized_name) {
|
||||
break;
|
||||
}
|
||||
suffix += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let cache = self.cache.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let con = cache.lock().map_err(|e| e.to_string())?;
|
||||
load_snapshot_to_duckdb(&con, &snapshot)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())??;
|
||||
|
||||
new_table_map.insert(sanitized_name, table_ref.clone());
|
||||
}
|
||||
|
||||
{
|
||||
let mut map = self.table_map.lock().map_err(|e| e.to_string())?;
|
||||
*map = new_table_map;
|
||||
}
|
||||
{
|
||||
let mut state = self.cache_state.lock().map_err(|e| e.to_string())?;
|
||||
*state = CacheState::Fresh;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for ExternalPool {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("ExternalPool")
|
||||
.field("source", &self.source.display_name())
|
||||
.field("cache_state", &self.cache_state)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
Vendored
-3
@@ -1,8 +1,5 @@
|
||||
pub mod duckdb_cache;
|
||||
pub mod traits;
|
||||
pub mod types;
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
pub use duckdb_cache::ExternalPool;
|
||||
pub use traits::ExternalTabularSource;
|
||||
pub use types::*;
|
||||
@@ -1,7 +1,3 @@
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
use chrono::{DateTime, Duration as ChronoDuration, NaiveDate, NaiveDateTime, NaiveTime, Utc};
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
use duckdb::types::{TimeUnit, Value, ValueRef};
|
||||
use futures::StreamExt;
|
||||
use mysql_async::prelude::Queryable;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -18,10 +14,6 @@ use std::sync::{
|
||||
Arc,
|
||||
};
|
||||
use std::time::Duration;
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
use tokio::task::JoinHandle;
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
use tokio::time::sleep;
|
||||
use tokio::time::timeout;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
@@ -30,8 +22,6 @@ use crate::database_capabilities;
|
||||
use crate::db;
|
||||
use crate::models::connection::{ConnectionConfig, DatabaseType};
|
||||
use crate::query_execution_sql::is_write_sql;
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
use crate::sql::starts_with_duckdb_result_sql_keyword;
|
||||
use crate::sql::{split_sql_batches, split_sql_statements, starts_with_executable_sql_keyword};
|
||||
use crate::sql_dialect::{resolve_for_db, CAP_TRANSACTIONAL_DDL};
|
||||
use crate::sql_risk::{classify_sql_risk_for_database, SqlRisk};
|
||||
@@ -45,10 +35,6 @@ pub const QUERY_CANCELED: &str = "Query canceled";
|
||||
const MONGO_SHELL_COMMAND_HINT: &str = "Use MongoDB shell-style commands, for example: db.collection.find({}).limit(100), db.collection.aggregate([]), db.collection.aggregate([], { explain: true }), db.version(), db.collection.countDocuments({}), db.collection.distinct(\"field\"), db.collection.getIndexes(), db.collection.createIndex({...}), or db.collection.insertOne({...}).";
|
||||
const SQL_OMITTED_ERROR_CONTEXT: &str =
|
||||
"SQL text omitted from user-facing error; enable debug SQL diagnostics for a redacted statement.";
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
const DUCKDB_INTERRUPT_DRAIN_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
const DUCKDB_DRAINING_MESSAGE: &str = "The previous DuckDB query is still stopping. Please try again shortly.";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PoolErrorAction {
|
||||
@@ -476,387 +462,6 @@ fn query_result_row_limit(max_rows: Option<usize>) -> usize {
|
||||
max_rows.unwrap_or(MAX_ROWS).max(1)
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
pub fn duckdb_execute(con: &duckdb::Connection, sql: &str) -> Result<db::QueryResult, String> {
|
||||
duckdb_execute_with_max_rows(con, sql, None)
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
fn duckdb_value_to_json(row: &duckdb::Row<'_>, idx: usize) -> serde_json::Value {
|
||||
let Ok(value_ref) = row.get_ref(idx) else {
|
||||
return serde_json::Value::Null;
|
||||
};
|
||||
match value_ref {
|
||||
ValueRef::Null => serde_json::Value::Null,
|
||||
ValueRef::Boolean(b) => serde_json::Value::Bool(b),
|
||||
ValueRef::TinyInt(i) => serde_json::Value::Number((i as i64).into()),
|
||||
ValueRef::SmallInt(i) => serde_json::Value::Number((i as i64).into()),
|
||||
ValueRef::Int(i) => serde_json::Value::Number((i as i64).into()),
|
||||
ValueRef::BigInt(i) => serde_json::Value::Number(i.into()),
|
||||
ValueRef::HugeInt(i) => serde_json::Value::String(i.to_string()),
|
||||
ValueRef::UTinyInt(i) => serde_json::Value::Number((i as u64).into()),
|
||||
ValueRef::USmallInt(i) => serde_json::Value::Number((i as u64).into()),
|
||||
ValueRef::UInt(i) => serde_json::Value::Number((i as u64).into()),
|
||||
ValueRef::UBigInt(i) => serde_json::Value::Number(i.into()),
|
||||
ValueRef::Float(f) => {
|
||||
serde_json::Number::from_f64(f as f64).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null)
|
||||
}
|
||||
ValueRef::Double(f) => {
|
||||
serde_json::Number::from_f64(f).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null)
|
||||
}
|
||||
ValueRef::Decimal(d) => serde_json::Value::String(d.to_string()),
|
||||
ValueRef::Date32(days) => {
|
||||
duckdb_date32_to_string(days).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
|
||||
}
|
||||
ValueRef::Time64(unit, value) => {
|
||||
duckdb_time64_to_string(unit, value).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
|
||||
}
|
||||
ValueRef::Timestamp(unit, value) => {
|
||||
duckdb_timestamp_to_string(unit, value).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
|
||||
}
|
||||
ValueRef::Text(bytes) => std::str::from_utf8(bytes)
|
||||
.map(|s| serde_json::Value::String(s.to_string()))
|
||||
.unwrap_or(serde_json::Value::Null),
|
||||
ValueRef::Blob(bytes) => {
|
||||
let hex: String = bytes.iter().map(|b| format!("{:02x}", b)).collect();
|
||||
serde_json::Value::String(format!("\\x{hex}"))
|
||||
}
|
||||
ValueRef::Interval { months, days, nanos } => {
|
||||
serde_json::Value::String(duckdb_interval_to_string(months, days, nanos))
|
||||
}
|
||||
ValueRef::List(..)
|
||||
| ValueRef::Array(..)
|
||||
| ValueRef::Struct(..)
|
||||
| ValueRef::Map(..)
|
||||
| ValueRef::Enum(..)
|
||||
| ValueRef::Union(..) => duckdb_owned_value_to_json(&value_ref.to_owned()),
|
||||
_ => duckdb_owned_value_to_json(&value_ref.to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
fn duckdb_owned_value_to_json(value: &Value) -> serde_json::Value {
|
||||
match value {
|
||||
Value::Null => serde_json::Value::Null,
|
||||
Value::Boolean(b) => serde_json::Value::Bool(*b),
|
||||
Value::TinyInt(i) => serde_json::Value::Number((*i as i64).into()),
|
||||
Value::SmallInt(i) => serde_json::Value::Number((*i as i64).into()),
|
||||
Value::Int(i) => serde_json::Value::Number((*i as i64).into()),
|
||||
Value::BigInt(i) => serde_json::Value::Number((*i).into()),
|
||||
Value::HugeInt(i) => serde_json::Value::String(i.to_string()),
|
||||
Value::UTinyInt(i) => serde_json::Value::Number((*i as u64).into()),
|
||||
Value::USmallInt(i) => serde_json::Value::Number((*i as u64).into()),
|
||||
Value::UInt(i) => serde_json::Value::Number((*i as u64).into()),
|
||||
Value::UBigInt(i) => serde_json::Value::Number((*i).into()),
|
||||
Value::Float(f) => {
|
||||
serde_json::Number::from_f64(*f as f64).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null)
|
||||
}
|
||||
Value::Double(f) => {
|
||||
serde_json::Number::from_f64(*f).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null)
|
||||
}
|
||||
Value::Decimal(d) => serde_json::Value::String(d.to_string()),
|
||||
Value::Timestamp(unit, value) => {
|
||||
duckdb_timestamp_to_string(*unit, *value).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
|
||||
}
|
||||
Value::Text(text) | Value::Enum(text) => serde_json::Value::String(text.clone()),
|
||||
Value::Blob(bytes) => {
|
||||
let hex: String = bytes.iter().map(|b| format!("{:02x}", b)).collect();
|
||||
serde_json::Value::String(format!("\\x{hex}"))
|
||||
}
|
||||
Value::Date32(days) => {
|
||||
duckdb_date32_to_string(*days).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
|
||||
}
|
||||
Value::Time64(unit, value) => {
|
||||
duckdb_time64_to_string(*unit, *value).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
|
||||
}
|
||||
Value::Interval { months, days, nanos } => {
|
||||
serde_json::Value::String(duckdb_interval_to_string(*months, *days, *nanos))
|
||||
}
|
||||
Value::List(values) | Value::Array(values) => {
|
||||
serde_json::Value::Array(values.iter().map(duckdb_owned_value_to_json).collect())
|
||||
}
|
||||
Value::Struct(entries) => serde_json::Value::Object(
|
||||
entries.iter().map(|(key, value)| (key.clone(), duckdb_owned_value_to_json(value))).collect(),
|
||||
),
|
||||
Value::Map(entries) => serde_json::Value::Array(
|
||||
entries
|
||||
.iter()
|
||||
.map(|(key, value)| {
|
||||
serde_json::json!({
|
||||
"key": duckdb_owned_value_to_json(key),
|
||||
"value": duckdb_owned_value_to_json(value),
|
||||
})
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
Value::Union(value) => duckdb_owned_value_to_json(value),
|
||||
_ => serde_json::Value::Null,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
fn duckdb_interval_to_string(months: i32, days: i32, nanos: i64) -> String {
|
||||
let mut parts = Vec::new();
|
||||
if months != 0 {
|
||||
let years = months / 12;
|
||||
let rem = months % 12;
|
||||
if years != 0 {
|
||||
parts.push(format!("{} year{}", years, if years.abs() != 1 { "s" } else { "" }));
|
||||
}
|
||||
if rem != 0 {
|
||||
parts.push(format!("{} mon{}", rem, if rem.abs() != 1 { "s" } else { "" }));
|
||||
}
|
||||
}
|
||||
if days != 0 {
|
||||
parts.push(format!("{} day{}", days, if days.abs() != 1 { "s" } else { "" }));
|
||||
}
|
||||
if nanos != 0 {
|
||||
let total_secs = nanos / 1_000_000_000;
|
||||
let hours = total_secs / 3600;
|
||||
let mins = (total_secs % 3600) / 60;
|
||||
let secs = total_secs % 60;
|
||||
let sub_nanos = (nanos % 1_000_000_000).unsigned_abs();
|
||||
if sub_nanos > 0 {
|
||||
parts.push(format!(
|
||||
"{:02}:{:02}:{:02}.{}",
|
||||
hours,
|
||||
mins,
|
||||
secs,
|
||||
format_temporal_without_empty_fraction(format!("0.{:09}", sub_nanos)).trim_start_matches("0.")
|
||||
));
|
||||
} else {
|
||||
parts.push(format!("{:02}:{:02}:{:02}", hours, mins, secs));
|
||||
}
|
||||
}
|
||||
if parts.is_empty() {
|
||||
"00:00:00".to_string()
|
||||
} else {
|
||||
parts.join(" ")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
fn duckdb_date32_to_string(days: i32) -> Option<String> {
|
||||
let epoch = NaiveDate::from_ymd_opt(1970, 1, 1)?;
|
||||
epoch.checked_add_signed(ChronoDuration::days(i64::from(days))).map(|date| date.to_string())
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
fn duckdb_time64_to_string(unit: TimeUnit, value: i64) -> Option<String> {
|
||||
let nanos = duckdb_time_unit_to_nanos(unit, value)?;
|
||||
let seconds = nanos.div_euclid(1_000_000_000);
|
||||
let nanos_remainder = nanos.rem_euclid(1_000_000_000) as u32;
|
||||
if !(0..86_400).contains(&seconds) {
|
||||
return None;
|
||||
}
|
||||
let time = NaiveTime::from_num_seconds_from_midnight_opt(seconds as u32, nanos_remainder)?;
|
||||
Some(format_temporal_without_empty_fraction(time.to_string()))
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
fn duckdb_timestamp_to_string(unit: TimeUnit, value: i64) -> Option<String> {
|
||||
let nanos = duckdb_time_unit_to_nanos(unit, value)?;
|
||||
let seconds = nanos.div_euclid(1_000_000_000);
|
||||
let nanos_remainder = nanos.rem_euclid(1_000_000_000) as u32;
|
||||
let dt: DateTime<Utc> = DateTime::from_timestamp(seconds, nanos_remainder)?;
|
||||
Some(format_naive_datetime(dt.naive_utc()))
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
fn duckdb_time_unit_to_nanos(unit: TimeUnit, value: i64) -> Option<i64> {
|
||||
match unit {
|
||||
TimeUnit::Second => value.checked_mul(1_000_000_000),
|
||||
TimeUnit::Millisecond => value.checked_mul(1_000_000),
|
||||
TimeUnit::Microsecond => value.checked_mul(1_000),
|
||||
TimeUnit::Nanosecond => Some(value),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
fn format_naive_datetime(value: NaiveDateTime) -> String {
|
||||
if value.and_utc().timestamp_subsec_nanos() == 0 {
|
||||
value.format("%Y-%m-%d %H:%M:%S").to_string()
|
||||
} else {
|
||||
format_temporal_without_empty_fraction(value.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
fn format_temporal_without_empty_fraction(value: String) -> String {
|
||||
if !value.contains('.') {
|
||||
return value;
|
||||
}
|
||||
let trimmed = value.trim_end_matches('0').trim_end_matches('.');
|
||||
trimmed.to_string()
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
pub fn duckdb_execute_with_max_rows(
|
||||
con: &duckdb::Connection,
|
||||
sql: &str,
|
||||
max_rows: Option<usize>,
|
||||
) -> Result<db::QueryResult, String> {
|
||||
let start = std::time::Instant::now();
|
||||
let row_limit = query_result_row_limit(max_rows);
|
||||
|
||||
if starts_with_duckdb_result_sql_keyword(sql) {
|
||||
let mut stmt = con.prepare(sql).map_err(|e| e.to_string())?;
|
||||
let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
|
||||
let stmt_ref = rows.as_ref().ok_or("DuckDB statement unavailable")?;
|
||||
let col_count = stmt_ref.column_count();
|
||||
let columns: Vec<String> = (0..col_count)
|
||||
.map(|i| stmt_ref.column_name(i).map(|s| s.to_string()).unwrap_or_else(|_| "?".to_string()))
|
||||
.collect();
|
||||
|
||||
let mut result_rows = Vec::new();
|
||||
while let Some(row) = rows.next().map_err(|e| e.to_string())? {
|
||||
let vals: Vec<serde_json::Value> = (0..col_count).map(|i| duckdb_value_to_json(row, i)).collect();
|
||||
result_rows.push(vals);
|
||||
if result_rows.len() > row_limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let truncated = result_rows.len() > row_limit;
|
||||
if truncated {
|
||||
result_rows.truncate(row_limit);
|
||||
}
|
||||
Ok(db::QueryResult {
|
||||
columns,
|
||||
column_types: Vec::new(),
|
||||
column_sortables: vec![],
|
||||
rows: result_rows,
|
||||
affected_rows: 0,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated,
|
||||
session_id: None,
|
||||
has_more: false,
|
||||
elasticsearch_raw_body: None,
|
||||
})
|
||||
} else {
|
||||
let affected = con.execute(sql, []).map_err(|e| e.to_string())?;
|
||||
Ok(db::QueryResult {
|
||||
columns: vec![],
|
||||
column_types: Vec::new(),
|
||||
column_sortables: vec![],
|
||||
rows: vec![],
|
||||
affected_rows: affected as u64,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
session_id: None,
|
||||
has_more: false,
|
||||
elasticsearch_raw_body: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
enum DuckDbTaskWait {
|
||||
Finished(Result<db::QueryResult, String>),
|
||||
Draining { error: String, task: JoinHandle<Result<db::QueryResult, String>> },
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
fn duckdb_join_result(
|
||||
result: Result<Result<db::QueryResult, String>, tokio::task::JoinError>,
|
||||
) -> Result<db::QueryResult, String> {
|
||||
result.map_err(|e| e.to_string())?
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
async fn interrupt_and_drain_duckdb_task(
|
||||
interrupt_handle: std::sync::Arc<duckdb::InterruptHandle>,
|
||||
mut task: JoinHandle<Result<db::QueryResult, String>>,
|
||||
error: String,
|
||||
) -> DuckDbTaskWait {
|
||||
interrupt_handle.interrupt();
|
||||
match timeout(DUCKDB_INTERRUPT_DRAIN_TIMEOUT, &mut task).await {
|
||||
Ok(result) => {
|
||||
let _ = result;
|
||||
DuckDbTaskWait::Finished(Err(error))
|
||||
}
|
||||
Err(_) => DuckDbTaskWait::Draining { error, task },
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
async fn wait_for_duckdb_task_with_interrupt_outcome(
|
||||
cancel_token: Option<CancellationToken>,
|
||||
timeout_duration: Option<Duration>,
|
||||
interrupt_handle: std::sync::Arc<duckdb::InterruptHandle>,
|
||||
mut task: JoinHandle<Result<db::QueryResult, String>>,
|
||||
) -> DuckDbTaskWait {
|
||||
match (cancel_token, timeout_duration) {
|
||||
(Some(token), Some(duration)) => {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => {
|
||||
interrupt_and_drain_duckdb_task(interrupt_handle, task, canceled_error()).await
|
||||
}
|
||||
result = &mut task => DuckDbTaskWait::Finished(duckdb_join_result(result)),
|
||||
_ = sleep(duration) => {
|
||||
interrupt_and_drain_duckdb_task(interrupt_handle, task, timeout_error()).await
|
||||
}
|
||||
}
|
||||
}
|
||||
(Some(token), None) => {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => {
|
||||
interrupt_and_drain_duckdb_task(interrupt_handle, task, canceled_error()).await
|
||||
}
|
||||
result = &mut task => DuckDbTaskWait::Finished(duckdb_join_result(result)),
|
||||
}
|
||||
}
|
||||
(None, Some(duration)) => {
|
||||
tokio::select! {
|
||||
result = &mut task => DuckDbTaskWait::Finished(duckdb_join_result(result)),
|
||||
_ = sleep(duration) => {
|
||||
interrupt_and_drain_duckdb_task(interrupt_handle, task, timeout_error()).await
|
||||
}
|
||||
}
|
||||
}
|
||||
(None, None) => DuckDbTaskWait::Finished(duckdb_join_result(task.await)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
async fn wait_for_duckdb_task_with_interrupt(
|
||||
cancel_token: Option<CancellationToken>,
|
||||
timeout_duration: Option<Duration>,
|
||||
interrupt_handle: std::sync::Arc<duckdb::InterruptHandle>,
|
||||
task: JoinHandle<Result<db::QueryResult, String>>,
|
||||
) -> Result<db::QueryResult, String> {
|
||||
match wait_for_duckdb_task_with_interrupt_outcome(cancel_token, timeout_duration, interrupt_handle, task).await {
|
||||
DuckDbTaskWait::Finished(result) => result,
|
||||
DuckDbTaskWait::Draining { error, .. } => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
pub(crate) fn duckdb_execute_for_database(
|
||||
con: &duckdb::Connection,
|
||||
attached_names: &[String],
|
||||
database: Option<&str>,
|
||||
sql: &str,
|
||||
max_rows: Option<usize>,
|
||||
) -> Result<db::QueryResult, String> {
|
||||
if let Some(database) = database.map(str::trim).filter(|database| !database.is_empty()) {
|
||||
let catalog = if database == "main" {
|
||||
crate::schema::duckdb_primary_catalog(con, attached_names)?
|
||||
} else {
|
||||
database.to_string()
|
||||
};
|
||||
con.execute_batch(&format!("USE {}", duckdb_quote_ident(&catalog))).map_err(|e| e.to_string())?;
|
||||
}
|
||||
duckdb_execute_with_max_rows(con, sql, max_rows)
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
fn duckdb_quote_ident(value: &str) -> String {
|
||||
format!("\"{}\"", value.replace('"', "\"\""))
|
||||
}
|
||||
|
||||
pub fn truncate_result(result: db::QueryResult) -> db::QueryResult {
|
||||
truncate_result_with_max_rows(result, None)
|
||||
}
|
||||
@@ -1166,11 +771,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
pub fn duckdb_draining_error() -> String {
|
||||
DUCKDB_DRAINING_MESSAGE.to_string()
|
||||
}
|
||||
|
||||
pub fn is_canceled(cancel_token: &Option<CancellationToken>) -> bool {
|
||||
cancel_token.as_ref().map(|token| token.is_cancelled()).unwrap_or(false)
|
||||
}
|
||||
@@ -1366,12 +966,10 @@ pub async fn do_execute(
|
||||
let _activity_touch = state.pool_activity_touch(pool_key);
|
||||
|
||||
let query_timeout = resolve_query_timeout(options.timeout_secs);
|
||||
let (_duckdb_attached_names, read_only_connection) = {
|
||||
let read_only_connection = {
|
||||
let configs = state.configs.read().await;
|
||||
let config = crate::connection::config_for_pool_key(pool_key, &configs);
|
||||
let attached = config.map(crate::db::duckdb_sql::config_attached_names).unwrap_or_default();
|
||||
let connection = config.filter(|config| config.read_only).map(|config| (config.name.clone(), config.db_type));
|
||||
(attached, connection)
|
||||
config.filter(|config| config.read_only).map(|config| (config.name.clone(), config.db_type))
|
||||
};
|
||||
let operation_budget = operation_budget_for_pool_key(state, pool_key, query_timeout).await;
|
||||
if let Some((name, database_type)) = read_only_connection {
|
||||
@@ -1383,49 +981,7 @@ pub async fn do_execute(
|
||||
let pool = connections.get(pool_key).ok_or("Connection not found")?;
|
||||
|
||||
let result = match pool {
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
PoolKind::DuckDb(con) => {
|
||||
let con = con.clone();
|
||||
if con.is_draining() {
|
||||
drop(connections);
|
||||
return Err(duckdb_draining_error());
|
||||
}
|
||||
let interrupt_handle = con.interrupt_handle();
|
||||
if let Some(ref execution_id) = options.execution_id {
|
||||
let cancel_interrupt_handle = interrupt_handle.clone();
|
||||
state.running_queries.register_interrupt(execution_id, move || {
|
||||
cancel_interrupt_handle.interrupt();
|
||||
});
|
||||
}
|
||||
let sql = sql.to_string();
|
||||
let database = database.map(str::to_string);
|
||||
let attached_names = _duckdb_attached_names;
|
||||
let max_rows = options.max_rows;
|
||||
drop(connections);
|
||||
let task_con = con.clone();
|
||||
let task = tokio::task::spawn_blocking(move || {
|
||||
let con = task_con.lock().map_err(|e| e.to_string())?;
|
||||
duckdb_execute_for_database(&con, &attached_names, database.as_deref(), &sql, max_rows)
|
||||
});
|
||||
let result =
|
||||
wait_for_duckdb_task_with_interrupt_outcome(cancel_token, query_timeout, interrupt_handle, task).await;
|
||||
match result {
|
||||
DuckDbTaskWait::Finished(result) => {
|
||||
if matches!(result.as_ref(), Err(err) if err == QUERY_CANCELED || is_dbx_query_timeout_error(&err.to_lowercase()))
|
||||
{
|
||||
con.mark_draining();
|
||||
state.spawn_duckdb_pool_cleanup(pool_key.to_string(), con);
|
||||
}
|
||||
result
|
||||
}
|
||||
DuckDbTaskWait::Draining { error, task } => {
|
||||
con.mark_draining();
|
||||
state.spawn_duckdb_draining_cleanup(pool_key.to_string(), con, task);
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
PoolKind::DuckDbWorker(client) => {
|
||||
let client = client.clone();
|
||||
if let Some(ref execution_id) = options.execution_id {
|
||||
@@ -1445,11 +1001,7 @@ pub async fn do_execute(
|
||||
drop(connections);
|
||||
client.execute(database, sql, max_rows, cancel_token, query_timeout).await
|
||||
}
|
||||
#[cfg(not(feature = "duckdb-bundled"))]
|
||||
PoolKind::DuckDb(_) => {
|
||||
return Err("DuckDB support is not compiled in this build".to_string());
|
||||
}
|
||||
#[cfg(not(feature = "duckdb-bundled"))]
|
||||
#[cfg(not(feature = "duckdb-sidecar"))]
|
||||
PoolKind::DuckDbWorker(_) => {
|
||||
return Err("DuckDB worker support is not compiled in this build".to_string());
|
||||
}
|
||||
@@ -1719,32 +1271,6 @@ pub async fn do_execute(
|
||||
}
|
||||
result
|
||||
}
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
PoolKind::ExternalTabular(ext_pool) => {
|
||||
if !starts_with_duckdb_result_sql_keyword(sql) {
|
||||
return Err("External data sources are read-only. Only SELECT queries are supported.".to_string());
|
||||
}
|
||||
let con = ext_pool.cache.clone();
|
||||
let interrupt_handle = con.lock().map_err(|e| e.to_string())?.interrupt_handle();
|
||||
if let Some(ref execution_id) = options.execution_id {
|
||||
let cancel_interrupt_handle = interrupt_handle.clone();
|
||||
state.running_queries.register_interrupt(execution_id, move || {
|
||||
cancel_interrupt_handle.interrupt();
|
||||
});
|
||||
}
|
||||
let sql = sql.to_string();
|
||||
let max_rows = options.max_rows;
|
||||
drop(connections);
|
||||
let task = tokio::task::spawn_blocking(move || {
|
||||
let con = con.lock().map_err(|e| e.to_string())?;
|
||||
duckdb_execute_with_max_rows(&con, &sql, max_rows)
|
||||
});
|
||||
wait_for_duckdb_task_with_interrupt(cancel_token, query_timeout, interrupt_handle, task).await
|
||||
}
|
||||
#[cfg(not(feature = "duckdb-bundled"))]
|
||||
PoolKind::ExternalTabular(_) => {
|
||||
Err("External data sources require DuckDB support. Rebuild with default features.".to_string())
|
||||
}
|
||||
PoolKind::ExternalDriver { config, session, .. } => {
|
||||
let config = config.clone();
|
||||
let session = session.clone();
|
||||
@@ -2769,7 +2295,6 @@ fn schema_diff_failure_outcome(
|
||||
}
|
||||
|
||||
fn pool_kind_has_transactional_path(pool: &PoolKind) -> bool {
|
||||
// DuckDb variants always exist (unit stubs when duckdb-bundled is off).
|
||||
match pool {
|
||||
PoolKind::Postgres(_)
|
||||
| PoolKind::Mysql(_, _)
|
||||
@@ -2783,14 +2308,12 @@ fn pool_kind_has_transactional_path(pool: &PoolKind) -> bool {
|
||||
PoolKind::MessageQueue
|
||||
| PoolKind::Nacos
|
||||
| PoolKind::HBase(_)
|
||||
| PoolKind::DuckDb(_)
|
||||
| PoolKind::DuckDbWorker(_)
|
||||
| PoolKind::Redis(_)
|
||||
| PoolKind::MongoDb(_)
|
||||
| PoolKind::Elasticsearch(_)
|
||||
| PoolKind::VectorDb(_)
|
||||
| PoolKind::InfluxDb(_)
|
||||
| PoolKind::ExternalTabular(_)
|
||||
| PoolKind::ExternalDriver { .. } => false,
|
||||
}
|
||||
}
|
||||
@@ -2980,25 +2503,12 @@ pub async fn execute_statements_in_transaction_on_pool(
|
||||
| PoolKind::SqlServer(_)
|
||||
| PoolKind::Agent(_) => TxPath::Explicit,
|
||||
PoolKind::MessageQueue | PoolKind::Nacos | PoolKind::HBase(_) => TxPath::None,
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
PoolKind::DuckDb(_)
|
||||
| PoolKind::DuckDbWorker(_)
|
||||
PoolKind::DuckDbWorker(_)
|
||||
| PoolKind::Redis(_)
|
||||
| PoolKind::MongoDb(_)
|
||||
| PoolKind::Elasticsearch(_)
|
||||
| PoolKind::VectorDb(_)
|
||||
| PoolKind::InfluxDb(_)
|
||||
| PoolKind::ExternalTabular(_)
|
||||
| PoolKind::ExternalDriver { .. } => TxPath::None,
|
||||
#[cfg(not(feature = "duckdb-bundled"))]
|
||||
PoolKind::DuckDb(_)
|
||||
| PoolKind::DuckDbWorker(_)
|
||||
| PoolKind::Redis(_)
|
||||
| PoolKind::MongoDb(_)
|
||||
| PoolKind::Elasticsearch(_)
|
||||
| PoolKind::VectorDb(_)
|
||||
| PoolKind::InfluxDb(_)
|
||||
| PoolKind::ExternalTabular(_)
|
||||
| PoolKind::ExternalDriver { .. } => TxPath::None,
|
||||
})
|
||||
};
|
||||
@@ -4744,192 +4254,6 @@ mod tests {
|
||||
assert_eq!(result.unwrap_err(), QUERY_CANCELED);
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn duckdb_timeout_interrupts_running_task_without_waiting_for_it_to_finish() {
|
||||
let con = std::sync::Arc::new(crate::db::duckdb_driver::DuckDbConnection::new(
|
||||
duckdb::Connection::open_in_memory().unwrap(),
|
||||
));
|
||||
let interrupt_handle = con.interrupt_handle();
|
||||
let running_con = con.clone();
|
||||
let task = tokio::task::spawn_blocking(move || {
|
||||
let con = running_con.lock().map_err(|e| e.to_string())?;
|
||||
duckdb_execute_with_max_rows(&con, "SELECT sum(sin(i::DOUBLE)) FROM range(10000000000) tbl(i)", None)
|
||||
});
|
||||
|
||||
let started = std::time::Instant::now();
|
||||
let result =
|
||||
wait_for_duckdb_task_with_interrupt(None, Some(Duration::from_millis(10)), interrupt_handle, task).await;
|
||||
|
||||
assert_eq!(result.unwrap_err(), timeout_error());
|
||||
assert!(started.elapsed() < Duration::from_secs(5));
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn duckdb_cancel_keeps_pool_draining_until_references_drop() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-query-duckdb-cancel-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let pool_key = "duckdb-1";
|
||||
let con = std::sync::Arc::new(crate::db::duckdb_driver::DuckDbConnection::new(
|
||||
duckdb::Connection::open_in_memory().unwrap(),
|
||||
));
|
||||
let extra_reference = con.clone();
|
||||
state.connections.write().await.insert(pool_key.to_string(), PoolKind::DuckDb(con));
|
||||
state.configs.write().await.insert(pool_key.to_string(), test_connection_config(DatabaseType::DuckDb));
|
||||
|
||||
let token = CancellationToken::new();
|
||||
let cancel_token = token.clone();
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
cancel_token.cancel();
|
||||
});
|
||||
|
||||
let result = do_execute(
|
||||
&state,
|
||||
pool_key,
|
||||
db::mysql::MySqlQueryDialect::default(),
|
||||
Some("main"),
|
||||
"SELECT sum(sin(i::DOUBLE)) FROM range(10000000000) tbl(i)",
|
||||
None,
|
||||
Some(token),
|
||||
QueryExecutionOptions::default(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(result.unwrap_err(), QUERY_CANCELED);
|
||||
let still_present = {
|
||||
let conns = state.connections.read().await;
|
||||
matches!(conns.get(pool_key), Some(PoolKind::DuckDb(current)) if current.is_draining())
|
||||
};
|
||||
assert!(still_present);
|
||||
|
||||
drop(extra_reference);
|
||||
// The draining-cleanup task (spawn_duckdb_draining_cleanup) cannot drop
|
||||
// the pool until the cancelled query's blocking DuckDB task has fully
|
||||
// unwound — the connection must stay alive while the query still holds
|
||||
// it. That unwind is near-instant on an idle runner but can exceed the
|
||||
// original 5s window under heavy CI load, flaking this test, so allow
|
||||
// ample headroom.
|
||||
timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
if !state.connections.read().await.contains_key(pool_key) {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("draining DuckDB pool should be removed after references drop");
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn duckdb_draining_pool_rejects_follow_up_query() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-query-duckdb-draining-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let pool_key = "duckdb-1";
|
||||
let con = std::sync::Arc::new(crate::db::duckdb_driver::DuckDbConnection::new(
|
||||
duckdb::Connection::open_in_memory().unwrap(),
|
||||
));
|
||||
con.mark_draining();
|
||||
state.connections.write().await.insert(pool_key.to_string(), PoolKind::DuckDb(con));
|
||||
state.configs.write().await.insert(pool_key.to_string(), test_connection_config(DatabaseType::DuckDb));
|
||||
|
||||
let result = do_execute(
|
||||
&state,
|
||||
pool_key,
|
||||
db::mysql::MySqlQueryDialect::default(),
|
||||
Some("main"),
|
||||
"SELECT 1",
|
||||
None,
|
||||
None,
|
||||
QueryExecutionOptions::default(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(result.unwrap_err(), duckdb_draining_error());
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn duckdb_draining_cleanup_removes_pool_after_task_finishes() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-query-duckdb-cleanup-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let pool_key = "duckdb-1";
|
||||
let con = std::sync::Arc::new(crate::db::duckdb_driver::DuckDbConnection::new(
|
||||
duckdb::Connection::open_in_memory().unwrap(),
|
||||
));
|
||||
con.mark_draining();
|
||||
state.connections.write().await.insert(pool_key.to_string(), PoolKind::DuckDb(con.clone()));
|
||||
|
||||
let task_con = con.clone();
|
||||
let task = tokio::task::spawn_blocking(move || {
|
||||
let _locked = task_con.lock().map_err(|e| e.to_string())?;
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
Ok(empty_query_result(0))
|
||||
});
|
||||
state.spawn_duckdb_draining_cleanup(pool_key.to_string(), con.clone(), task);
|
||||
|
||||
assert!(state.connections.read().await.contains_key(pool_key));
|
||||
drop(con);
|
||||
timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if !state.connections.read().await.contains_key(pool_key) {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("draining cleanup should remove the DuckDB pool");
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn duckdb_cleanup_keeps_draining_pool_while_extra_reference_exists() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-query-duckdb-cleanup-ref-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let pool_key = "duckdb-1";
|
||||
let con = std::sync::Arc::new(crate::db::duckdb_driver::DuckDbConnection::new(
|
||||
duckdb::Connection::open_in_memory().unwrap(),
|
||||
));
|
||||
con.mark_draining();
|
||||
state.connections.write().await.insert(pool_key.to_string(), PoolKind::DuckDb(con.clone()));
|
||||
|
||||
let extra_reference = con.clone();
|
||||
let task = tokio::task::spawn_blocking(|| Ok(empty_query_result(0)));
|
||||
state.spawn_duckdb_draining_cleanup(pool_key.to_string(), con.clone(), task);
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
|
||||
let still_present = {
|
||||
let conns = state.connections.read().await;
|
||||
matches!(conns.get(pool_key), Some(PoolKind::DuckDb(current)) if current.is_draining())
|
||||
};
|
||||
assert!(still_present);
|
||||
|
||||
drop(extra_reference);
|
||||
drop(con);
|
||||
timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if !state.connections.read().await.contains_key(pool_key) {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("draining cleanup should remove the DuckDB pool after extra refs drop");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn db_operation_budget_from_config() {
|
||||
let budget = DbOperationBudget::from_config(10, Some(30));
|
||||
@@ -5162,150 +4486,6 @@ mod tests {
|
||||
assert_eq!(pool_error_action(Some(DatabaseType::Dameng), dameng_err), PoolErrorAction::ReconnectAndRetry);
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[test]
|
||||
fn duckdb_execute_preserves_double_precision() {
|
||||
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
|
||||
let result = duckdb_execute(
|
||||
&con,
|
||||
"SELECT 12.34567::DOUBLE AS sample, 0.5::DOUBLE AS half, 99.99::DOUBLE AS price, 1.0::DOUBLE AS one",
|
||||
)
|
||||
.expect("execute double query");
|
||||
|
||||
assert_eq!(result.columns, vec!["sample", "half", "price", "one"]);
|
||||
let row = &result.rows[0];
|
||||
assert_eq!(row[0], serde_json::json!(12.34567));
|
||||
assert_eq!(row[1], serde_json::json!(0.5));
|
||||
assert_eq!(row[2], serde_json::json!(99.99));
|
||||
assert_eq!(row[3], serde_json::json!(1.0));
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[test]
|
||||
fn duckdb_execute_create_insert_select_double() {
|
||||
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
|
||||
con.execute_batch("CREATE TABLE tmp1 (tmp_double DOUBLE)").expect("create table");
|
||||
con.execute_batch("INSERT INTO tmp1 VALUES (45.678), (12.345), (99.999)").expect("insert");
|
||||
|
||||
let result = duckdb_execute(&con, "SELECT tmp_double FROM tmp1 ORDER BY tmp_double").expect("select doubles");
|
||||
|
||||
assert_eq!(result.rows.len(), 3);
|
||||
assert_eq!(result.rows[0][0], serde_json::json!(12.345));
|
||||
assert_eq!(result.rows[1][0], serde_json::json!(45.678));
|
||||
assert_eq!(result.rows[2][0], serde_json::json!(99.999));
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[test]
|
||||
fn duckdb_execute_returns_rows_for_from_first_query() {
|
||||
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
|
||||
con.execute_batch("CREATE TABLE users (id INTEGER, name VARCHAR)").expect("create table");
|
||||
con.execute_batch("INSERT INTO users VALUES (2, 'Grace'), (1, 'Ada')").expect("insert");
|
||||
|
||||
let result = duckdb_execute(&con, "FROM users ORDER BY id").expect("execute from-first query");
|
||||
|
||||
assert_eq!(result.columns, vec!["id", "name"]);
|
||||
assert_eq!(result.rows.len(), 2);
|
||||
assert_eq!(result.rows[0], vec![serde_json::json!(1), serde_json::json!("Ada")]);
|
||||
assert_eq!(result.rows[1], vec![serde_json::json!(2), serde_json::json!("Grace")]);
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[test]
|
||||
fn duckdb_execute_returns_rows_for_summarize_query() {
|
||||
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
|
||||
con.execute_batch("CREATE TABLE metrics (value INTEGER)").expect("create table");
|
||||
con.execute_batch("INSERT INTO metrics VALUES (1), (2), (NULL)").expect("insert");
|
||||
|
||||
let result = duckdb_execute(&con, "SUMMARIZE metrics").expect("execute summarize query");
|
||||
|
||||
assert!(!result.columns.is_empty());
|
||||
assert!(!result.rows.is_empty());
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[test]
|
||||
fn duckdb_execute_handles_various_types() {
|
||||
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
|
||||
let result = duckdb_execute(
|
||||
&con,
|
||||
"SELECT 42 AS int_val, true AS bool_val, 'hello' AS text_val, 3.14::FLOAT AS float_val, 123456789012345::BIGINT AS big_val",
|
||||
)
|
||||
.expect("execute mixed types query");
|
||||
|
||||
let row = &result.rows[0];
|
||||
assert_eq!(row[0], serde_json::json!(42));
|
||||
assert_eq!(row[1], serde_json::json!(true));
|
||||
assert_eq!(row[2], serde_json::Value::String("hello".to_string()));
|
||||
assert!(row[3].is_number());
|
||||
assert_eq!(row[4], serde_json::json!(123456789012345_i64));
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[test]
|
||||
fn duckdb_execute_returns_list_values_as_json_arrays() {
|
||||
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
|
||||
let result = duckdb_execute(&con, "SELECT ['a','b','c','d'];").expect("execute list query");
|
||||
|
||||
assert_eq!(result.rows, vec![vec![serde_json::json!(["a", "b", "c", "d"])]]);
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[test]
|
||||
fn duckdb_execute_preserves_nulls_inside_list_values() {
|
||||
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
|
||||
let result = duckdb_execute(&con, "SELECT [1, NULL, 3] AS items;").expect("execute nullable list query");
|
||||
|
||||
assert_eq!(result.columns, vec!["items"]);
|
||||
assert_eq!(result.rows, vec![vec![serde_json::json!([1, null, 3])]]);
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[test]
|
||||
fn duckdb_execute_returns_nested_complex_values_as_json() {
|
||||
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
|
||||
let result = duckdb_execute(
|
||||
&con,
|
||||
"SELECT {'name': 'Ada', 'scores': [10, 20]} AS profile, MAP(['x', 'y'], [1, 2]) AS lookup, [1, 2, 3]::INTEGER[3] AS fixed_items",
|
||||
)
|
||||
.expect("execute complex values query");
|
||||
|
||||
assert_eq!(result.columns, vec!["profile", "lookup", "fixed_items"]);
|
||||
assert_eq!(
|
||||
result.rows,
|
||||
vec![vec![
|
||||
serde_json::json!({ "name": "Ada", "scores": [10, 20] }),
|
||||
serde_json::json!([
|
||||
{ "key": "x", "value": 1 },
|
||||
{ "key": "y", "value": 2 },
|
||||
]),
|
||||
serde_json::json!([1, 2, 3]),
|
||||
]]
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[test]
|
||||
fn duckdb_execute_formats_temporal_values_by_column_type() {
|
||||
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
|
||||
let result = duckdb_execute(
|
||||
&con,
|
||||
"SELECT DATE '2026-05-14' AS d, TIME '16:58:15' AS t, TIMESTAMP '2026-05-14 16:58:15.0' AS ts, NULL::TIMESTAMP AS nts",
|
||||
)
|
||||
.expect("execute temporal query");
|
||||
|
||||
assert_eq!(result.columns, vec!["d", "t", "ts", "nts"]);
|
||||
assert_eq!(
|
||||
result.rows,
|
||||
vec![vec![
|
||||
serde_json::Value::String("2026-05-14".to_string()),
|
||||
serde_json::Value::String("16:58:15".to_string()),
|
||||
serde_json::Value::String("2026-05-14 16:58:15".to_string()),
|
||||
serde_json::Value::Null,
|
||||
]]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_driver_query_params_include_database_and_schema_context() {
|
||||
let config = ConnectionConfig {
|
||||
|
||||
+12
-912
File diff suppressed because it is too large.
Load diff
@@ -1,203 +0,0 @@
|
||||
use crate::connection::AppState;
|
||||
use crate::db;
|
||||
|
||||
pub fn duckdb_query_tables(con: &duckdb::Connection) -> Result<Vec<db::TableInfo>, String> {
|
||||
duckdb_query_tables_in_database(con, "main", "main")
|
||||
}
|
||||
|
||||
pub fn duckdb_query_tables_in_database(
|
||||
con: &duckdb::Connection,
|
||||
database: &str,
|
||||
schema: &str,
|
||||
) -> Result<Vec<db::TableInfo>, String> {
|
||||
duckdb_query_tables_in_database_with_attached(con, database, schema, &[])
|
||||
}
|
||||
|
||||
pub fn duckdb_query_tables_in_database_with_attached(
|
||||
con: &duckdb::Connection,
|
||||
database: &str,
|
||||
schema: &str,
|
||||
attached_names: &[String],
|
||||
) -> Result<Vec<db::TableInfo>, String> {
|
||||
let database = duckdb_catalog_name(con, database, attached_names)?;
|
||||
let mut stmt = con
|
||||
.prepare(
|
||||
"SELECT table_name, table_type FROM information_schema.tables WHERE table_catalog = ? AND table_schema = ? ORDER BY table_name",
|
||||
)
|
||||
.map_err(|e| e.to_string())?;
|
||||
let rows = stmt
|
||||
.query_map((database.as_str(), schema), |row| {
|
||||
Ok(db::TableInfo {
|
||||
name: row.get::<_, String>(0)?,
|
||||
table_type: row.get::<_, String>(1)?,
|
||||
comment: None,
|
||||
parent_schema: None,
|
||||
parent_name: None,
|
||||
})
|
||||
})
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(rows.filter_map(|r| r.ok()).collect())
|
||||
}
|
||||
|
||||
pub fn duckdb_attach_database(con: &duckdb::Connection, name: &str, path: &str) -> Result<(), String> {
|
||||
let name = name.trim();
|
||||
let path = path.trim();
|
||||
if name.is_empty() || path.is_empty() {
|
||||
return Err("DuckDB attached database name and path are required".to_string());
|
||||
}
|
||||
let sql = format!("ATTACH {} AS {}", duckdb_quote_string(path), duckdb_quote_ident(name));
|
||||
con.execute_batch(&sql).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub fn duckdb_list_databases(con: &duckdb::Connection) -> Result<Vec<db::DatabaseInfo>, String> {
|
||||
duckdb_list_databases_with_attached(con, &[])
|
||||
}
|
||||
|
||||
pub fn duckdb_list_databases_with_attached(
|
||||
con: &duckdb::Connection,
|
||||
attached_names: &[String],
|
||||
) -> Result<Vec<db::DatabaseInfo>, String> {
|
||||
let primary = duckdb_primary_catalog(con, attached_names)?;
|
||||
let mut stmt = con.prepare("SHOW DATABASES").map_err(|e| e.to_string())?;
|
||||
let rows = stmt
|
||||
.query_map([], |row| {
|
||||
let name = row.get::<_, String>(0)?;
|
||||
Ok(db::DatabaseInfo { name: if name == primary { "main".to_string() } else { name } })
|
||||
})
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(rows.filter_map(|row| row.ok()).collect())
|
||||
}
|
||||
|
||||
pub fn duckdb_list_schemas(con: &duckdb::Connection, database: &str) -> Result<Vec<String>, String> {
|
||||
duckdb_list_schemas_with_attached(con, database, &[])
|
||||
}
|
||||
|
||||
pub fn duckdb_list_schemas_with_attached(
|
||||
con: &duckdb::Connection,
|
||||
database: &str,
|
||||
attached_names: &[String],
|
||||
) -> Result<Vec<String>, String> {
|
||||
let database = duckdb_catalog_name(con, database, attached_names)?;
|
||||
let mut stmt = con
|
||||
.prepare(
|
||||
"SELECT schema_name FROM information_schema.schemata WHERE catalog_name = ? AND schema_name NOT IN ('information_schema', 'pg_catalog') ORDER BY schema_name",
|
||||
)
|
||||
.map_err(|e| e.to_string())?;
|
||||
let rows = stmt.query_map([database.as_str()], |row| row.get::<_, String>(0)).map_err(|e| e.to_string())?;
|
||||
Ok(rows.filter_map(|r| r.ok()).collect())
|
||||
}
|
||||
|
||||
fn duckdb_catalog_name(con: &duckdb::Connection, database: &str, attached_names: &[String]) -> Result<String, String> {
|
||||
if database.trim().is_empty() || database == "main" {
|
||||
return duckdb_primary_catalog(con, attached_names);
|
||||
}
|
||||
Ok(database.to_string())
|
||||
}
|
||||
|
||||
pub fn duckdb_primary_catalog(con: &duckdb::Connection, attached_names: &[String]) -> Result<String, String> {
|
||||
if attached_names.is_empty() {
|
||||
return duckdb_current_database(con);
|
||||
}
|
||||
let attached: std::collections::HashSet<String> = attached_names.iter().map(|name| name.to_lowercase()).collect();
|
||||
let mut stmt = con.prepare("SHOW DATABASES").map_err(|e| e.to_string())?;
|
||||
let rows = stmt.query_map([], |row| row.get::<_, String>(0)).map_err(|e| e.to_string())?;
|
||||
for row in rows {
|
||||
let name = row.map_err(|e| e.to_string())?;
|
||||
if !attached.contains(&name.to_lowercase()) {
|
||||
return Ok(name);
|
||||
}
|
||||
}
|
||||
duckdb_current_database(con)
|
||||
}
|
||||
|
||||
fn duckdb_current_database(con: &duckdb::Connection) -> Result<String, String> {
|
||||
con.query_row("SELECT current_database()", [], |row| row.get::<_, String>(0)).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
fn duckdb_quote_ident(value: &str) -> String {
|
||||
format!("\"{}\"", value.replace('"', "\"\""))
|
||||
}
|
||||
|
||||
fn duckdb_quote_string(value: &str) -> String {
|
||||
format!("'{}'", value.replace('\'', "''"))
|
||||
}
|
||||
|
||||
pub fn duckdb_query_columns(con: &duckdb::Connection, table: &str) -> Result<Vec<db::ColumnInfo>, String> {
|
||||
duckdb_query_columns_in_database(con, "main", "main", table)
|
||||
}
|
||||
|
||||
pub fn duckdb_query_columns_in_database(
|
||||
con: &duckdb::Connection,
|
||||
database: &str,
|
||||
schema: &str,
|
||||
table: &str,
|
||||
) -> Result<Vec<db::ColumnInfo>, String> {
|
||||
duckdb_query_columns_in_database_with_attached(con, database, schema, table, &[])
|
||||
}
|
||||
|
||||
pub fn duckdb_query_columns_in_database_with_attached(
|
||||
con: &duckdb::Connection,
|
||||
database: &str,
|
||||
schema: &str,
|
||||
table: &str,
|
||||
attached_names: &[String],
|
||||
) -> Result<Vec<db::ColumnInfo>, String> {
|
||||
let database = duckdb_catalog_name(con, database, attached_names)?;
|
||||
let mut pk_stmt = con
|
||||
.prepare(
|
||||
"SELECT kcu.column_name
|
||||
FROM information_schema.table_constraints tc
|
||||
JOIN information_schema.key_column_usage kcu
|
||||
ON tc.constraint_name = kcu.constraint_name
|
||||
AND tc.table_schema = kcu.table_schema
|
||||
AND tc.table_name = kcu.table_name
|
||||
WHERE tc.constraint_type = 'PRIMARY KEY'
|
||||
AND tc.table_catalog = ?
|
||||
AND tc.table_schema = ?
|
||||
AND tc.table_name = ?
|
||||
ORDER BY kcu.ordinal_position",
|
||||
)
|
||||
.map_err(|e| e.to_string())?;
|
||||
let pk_rows = pk_stmt
|
||||
.query_map((database.as_str(), schema, table), |row| row.get::<_, String>(0))
|
||||
.map_err(|e| e.to_string())?;
|
||||
let primary_keys: std::collections::HashSet<String> = pk_rows.filter_map(|r| r.ok()).collect();
|
||||
|
||||
let mut stmt = con
|
||||
.prepare(
|
||||
"SELECT column_name, data_type, is_nullable, column_default
|
||||
FROM information_schema.columns
|
||||
WHERE table_catalog = ? AND table_schema = ? AND table_name = ?
|
||||
ORDER BY ordinal_position",
|
||||
)
|
||||
.map_err(|e| e.to_string())?;
|
||||
let rows = stmt
|
||||
.query_map((database.as_str(), schema, table), |row| {
|
||||
let name = row.get::<_, String>(0)?;
|
||||
Ok(db::ColumnInfo {
|
||||
is_primary_key: primary_keys.contains(&name),
|
||||
name,
|
||||
data_type: row.get::<_, String>(1)?,
|
||||
is_nullable: row.get::<_, String>(2).unwrap_or_default() == "YES",
|
||||
column_default: row.get::<_, Option<String>>(3)?,
|
||||
extra: None,
|
||||
comment: None,
|
||||
numeric_precision: None,
|
||||
numeric_scale: None,
|
||||
character_maximum_length: None,
|
||||
..Default::default()
|
||||
})
|
||||
})
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(rows.filter_map(|r| r.ok()).collect())
|
||||
}
|
||||
|
||||
pub(super) async fn duckdb_attached_database_names(state: &AppState, connection_id: &str) -> Vec<String> {
|
||||
state
|
||||
.configs
|
||||
.read()
|
||||
.await
|
||||
.get(connection_id)
|
||||
.map(crate::db::duckdb_sql::config_attached_names)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
@@ -6,7 +6,7 @@ pub(super) enum StructureDialect {
|
||||
Doris,
|
||||
Postgres,
|
||||
Sqlite,
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
DuckDb,
|
||||
SqlServer,
|
||||
Oracle,
|
||||
@@ -171,7 +171,7 @@ pub(super) fn capabilities_for(database_type: Option<DatabaseType>) -> TableStru
|
||||
index_filter: true,
|
||||
..base
|
||||
},
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
Some(DatabaseType::DuckDb) => TableStructureCapabilities {
|
||||
dialect: StructureDialect::DuckDb,
|
||||
add_column: true,
|
||||
@@ -305,7 +305,7 @@ pub(super) fn dialect_label(dialect: StructureDialect) -> String {
|
||||
StructureDialect::Doris => "doris",
|
||||
StructureDialect::Postgres => "postgres",
|
||||
StructureDialect::Sqlite => "sqlite",
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
StructureDialect::DuckDb => "duckdb",
|
||||
StructureDialect::SqlServer => "sqlserver",
|
||||
StructureDialect::Oracle => "oracle",
|
||||
@@ -326,7 +326,7 @@ pub(super) fn database_type_for_dialect(dialect: StructureDialect) -> Option<Dat
|
||||
StructureDialect::Doris => Some(DatabaseType::Doris),
|
||||
StructureDialect::Postgres => Some(DatabaseType::Postgres),
|
||||
StructureDialect::Sqlite => Some(DatabaseType::Sqlite),
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
StructureDialect::DuckDb => Some(DatabaseType::DuckDb),
|
||||
StructureDialect::SqlServer => Some(DatabaseType::SqlServer),
|
||||
StructureDialect::Oracle => Some(DatabaseType::Oracle),
|
||||
|
||||
@@ -2465,7 +2465,7 @@ fn sqlserver_existing_column_identity_change_warns_without_unchanged_foreign_key
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
#[test]
|
||||
fn builds_duckdb_create_table_statements() {
|
||||
let mut name = column("name");
|
||||
|
||||
+10
-193
@@ -10,8 +10,6 @@ use crate::models::connection::DatabaseType;
|
||||
use crate::object_source_sql::{build_executable_object_source_statements, EditableObjectSourceSqlInput};
|
||||
use crate::query::{agent_execute_query_params, pool_error_action, PoolErrorAction, QueryExecutionOptions};
|
||||
use crate::sql::split_sql_statements;
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
use crate::sql::starts_with_executable_sql_keyword;
|
||||
use crate::sql_dialect::{qualified_transfer_table, quote_transfer_identifier};
|
||||
|
||||
static CANCELLED: std::sync::LazyLock<RwLock<HashSet<String>>> =
|
||||
@@ -3048,89 +3046,12 @@ async fn execute_on_pool_once(
|
||||
);
|
||||
client.execute_query(params).await
|
||||
}
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
PoolKind::DuckDb(con) => {
|
||||
let con = con.clone();
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
PoolKind::DuckDbWorker(client) => {
|
||||
let client = client.clone();
|
||||
let sql = sql.to_string();
|
||||
drop(connections);
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let con = con.lock().map_err(|e| e.to_string())?;
|
||||
if max_rows.is_some()
|
||||
&& starts_with_executable_sql_keyword(&sql, &["SELECT", "SHOW", "DESCRIBE", "WITH", "PRAGMA"])
|
||||
{
|
||||
return crate::query::duckdb_execute_with_max_rows(&con, &sql, max_rows);
|
||||
}
|
||||
let start = std::time::Instant::now();
|
||||
if starts_with_executable_sql_keyword(&sql, &["SELECT", "SHOW", "DESCRIBE", "WITH", "PRAGMA"]) {
|
||||
let mut stmt = con.prepare(&sql).map_err(|e| e.to_string())?;
|
||||
let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
|
||||
let stmt_ref = rows.as_ref().ok_or("DuckDB statement unavailable")?;
|
||||
let col_count = stmt_ref.column_count();
|
||||
let columns: Vec<String> = (0..col_count)
|
||||
.map(|i| stmt_ref.column_name(i).map(|s| s.to_string()).unwrap_or_else(|_| "?".to_string()))
|
||||
.collect();
|
||||
let mut result_rows = Vec::new();
|
||||
while let Some(row) = rows.next().map_err(|e| e.to_string())? {
|
||||
let vals: Vec<serde_json::Value> = (0..col_count)
|
||||
.map(|i| {
|
||||
row.get::<_, String>(i)
|
||||
.map(serde_json::Value::String)
|
||||
.or_else(|_| row.get::<_, i64>(i).map(|v| serde_json::Value::Number(v.into())))
|
||||
.or_else(|_| {
|
||||
row.get::<_, f64>(i).map(|v| {
|
||||
serde_json::Number::from_f64(v)
|
||||
.map(serde_json::Value::Number)
|
||||
.unwrap_or(serde_json::Value::Null)
|
||||
})
|
||||
})
|
||||
.or_else(|_| row.get::<_, bool>(i).map(serde_json::Value::Bool))
|
||||
.unwrap_or(serde_json::Value::Null)
|
||||
})
|
||||
.collect();
|
||||
result_rows.push(vals);
|
||||
}
|
||||
Ok(db::QueryResult {
|
||||
columns,
|
||||
column_types: Vec::new(),
|
||||
column_sortables: vec![],
|
||||
rows: result_rows,
|
||||
affected_rows: 0,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
session_id: None,
|
||||
has_more: false,
|
||||
elasticsearch_raw_body: None,
|
||||
})
|
||||
} else {
|
||||
let affected = con.execute(&sql, []).map_err(|e| e.to_string())?;
|
||||
Ok(db::QueryResult {
|
||||
columns: vec![],
|
||||
column_types: Vec::new(),
|
||||
column_sortables: vec![],
|
||||
rows: vec![],
|
||||
affected_rows: affected as u64,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
session_id: None,
|
||||
has_more: false,
|
||||
elasticsearch_raw_body: None,
|
||||
})
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
}
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
PoolKind::ExternalTabular(ext_pool) => {
|
||||
let con = ext_pool.cache.clone();
|
||||
let sql = sql.to_string();
|
||||
drop(connections);
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let con = con.lock().map_err(|e| e.to_string())?;
|
||||
crate::query::duckdb_execute_with_max_rows(&con, &sql, max_rows)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
client.execute(None, sql, max_rows, None, None).await
|
||||
}
|
||||
_ => Err("Unsupported database type for transfer".to_string()),
|
||||
}
|
||||
@@ -3161,32 +3082,14 @@ pub async fn get_columns_for_transfer(
|
||||
) -> Result<Vec<db::ColumnInfo>, String> {
|
||||
let connections = state.connections.read().await;
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
if let Some(PoolKind::DuckDb(con)) = connections.get(pool_key) {
|
||||
let con = con.clone();
|
||||
drop(connections);
|
||||
let table = table.to_string();
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
if let Some(PoolKind::DuckDbWorker(client)) = connections.get(pool_key) {
|
||||
let client = client.clone();
|
||||
let database = database.to_string();
|
||||
let schema = schema.to_string();
|
||||
return tokio::task::spawn_blocking(move || {
|
||||
let con = con.lock().map_err(|e| e.to_string())?;
|
||||
crate::schema::duckdb_query_columns_in_database(&con, "main", &schema, &table)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
if let Some(PoolKind::ExternalTabular(ext_pool)) = connections.get(pool_key) {
|
||||
let con = ext_pool.cache.clone();
|
||||
drop(connections);
|
||||
let table = table.to_string();
|
||||
let schema = schema.to_string();
|
||||
return tokio::task::spawn_blocking(move || {
|
||||
let con = con.lock().map_err(|e| e.to_string())?;
|
||||
crate::schema::duckdb_query_columns_in_database(&con, "main", &schema, &table)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
drop(connections);
|
||||
return client.list_columns(database, schema, table).await;
|
||||
}
|
||||
|
||||
if let Some(PoolKind::ClickHouse(client)) = connections.get(pool_key) {
|
||||
@@ -5029,73 +4932,7 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
use crate::connection::{AppState, PoolKind};
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
use crate::models::connection::default_redis_key_separator;
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
use crate::storage::Storage;
|
||||
use serde_json::json;
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
use std::sync::Arc;
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
fn duckdb_test_config(id: &str) -> crate::models::connection::ConnectionConfig {
|
||||
crate::models::connection::ConnectionConfig {
|
||||
id: id.to_string(),
|
||||
name: id.to_string(),
|
||||
note: String::new(),
|
||||
db_type: DatabaseType::DuckDb,
|
||||
driver_profile: None,
|
||||
driver_label: None,
|
||||
url_params: None,
|
||||
agent_java_options: Vec::new(),
|
||||
host: ":memory:".to_string(),
|
||||
port: 0,
|
||||
username: String::new(),
|
||||
password: String::new(),
|
||||
database: None,
|
||||
visible_databases: None,
|
||||
visible_schemas: None,
|
||||
show_system_schemas: false,
|
||||
attached_databases: Vec::new(),
|
||||
init_script: None,
|
||||
color: None,
|
||||
transport_layers: Vec::new(),
|
||||
connect_timeout_secs: 5,
|
||||
query_timeout_secs: 30,
|
||||
idle_timeout_secs: 60,
|
||||
keepalive_interval_secs: 0,
|
||||
ssl: false,
|
||||
ca_cert_path: String::new(),
|
||||
client_cert_path: String::new(),
|
||||
client_key_path: String::new(),
|
||||
sysdba: false,
|
||||
oracle_connection_type: None,
|
||||
connection_string: None,
|
||||
redis_connection_mode: None,
|
||||
redis_sentinel_master: String::new(),
|
||||
redis_sentinel_nodes: String::new(),
|
||||
redis_sentinel_username: String::new(),
|
||||
redis_sentinel_password: String::new(),
|
||||
redis_sentinel_tls: false,
|
||||
redis_cluster_nodes: String::new(),
|
||||
redis_key_separator: default_redis_key_separator(),
|
||||
redis_scan_page_size: None,
|
||||
redis_database_aliases: Default::default(),
|
||||
etcd_endpoints: String::new(),
|
||||
gbase_server: String::new(),
|
||||
informix_server: String::new(),
|
||||
external_config: None,
|
||||
jdbc_driver_class: None,
|
||||
jdbc_driver_paths: Vec::new(),
|
||||
one_time: false,
|
||||
read_only: false,
|
||||
is_production: false,
|
||||
production_databases: vec![],
|
||||
database_info: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn test_column(name: &str, data_type: &str) -> db::ColumnInfo {
|
||||
db::ColumnInfo {
|
||||
@@ -6863,26 +6700,6 @@ SELECT 1 FROM dual"#
|
||||
assert!(statements[0].contains("ON DUPLICATE KEY UPDATE"));
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
#[tokio::test]
|
||||
async fn duckdb_transfer_columns_use_requested_schema() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-transfer-test-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let con = duckdb::Connection::open_in_memory().unwrap();
|
||||
con.execute_batch("CREATE SCHEMA analytics; CREATE TABLE analytics.items(id INTEGER);").unwrap();
|
||||
|
||||
let state = AppState::new(storage);
|
||||
let con = Arc::new(crate::db::duckdb_driver::DuckDbConnection::new(con));
|
||||
state.connections.write().await.insert("duckdb-1".to_string(), PoolKind::DuckDb(con));
|
||||
state.configs.write().await.insert("duckdb-1".to_string(), duckdb_test_config("duckdb-1"));
|
||||
|
||||
let columns =
|
||||
get_columns_for_transfer(&state, "duckdb-1", "duckdb-1", "main", "analytics", "items").await.unwrap();
|
||||
|
||||
assert_eq!(columns.iter().map(|c| c.name.as_str()).collect::<Vec<_>>(), vec!["id"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn database_from_pool_key_handles_session_scoped_keys() {
|
||||
assert_eq!(database_from_pool_key("conn:analytics"), Some("analytics"));
|
||||
|
||||
Reference in new issue
Block a user