feat(duckdb): move support to standalone driver

This commit is contained in:
t8y2 committed 2026-07-30 00:50:39 +08:00
1 parent ed018dbc77
commit d347c77c3d
76 files changed
+5400 -8258

No files matched your search

+1
View File
@@ -43,6 +43,7 @@ function fileContainsCommonDependency(path, moduleExists, readModuleFile) {
} }
const nativeDriverDirectories = { const nativeDriverDirectories = {
duckdb: "duckdb",
oracle: "oracle-go", oracle: "oracle-go",
kingbase: "kingbase-go", kingbase: "kingbase-go",
}; };
@@ -0,0 +1,31 @@
import assert from "node:assert/strict";
import test from "node:test";
import { evaluateAgentVersionBump } from "./bump-agent-versions.mjs";
const moduleExists = (path) => path === "agents/drivers/duckdb";
test("keeps the manually registered first DuckDB driver version", () => {
const result = evaluateAgentVersionBump({
versions: { duckdb: "0.1.0" },
prevVersions: {},
changedFiles: ["agents/versions.json", "agents/drivers/duckdb/src/main.rs"],
moduleExists,
readModuleFile: () => "",
});
assert.equal(result.versions.duckdb, "0.1.0");
assert.equal(result.changed, false);
assert.match(result.logs.join("\n"), /new module version kept at 0\.1\.0/);
});
test("bumps DuckDB after its initial release", () => {
const result = evaluateAgentVersionBump({
versions: { duckdb: "0.1.0" },
changedFiles: ["agents/drivers/duckdb/src/main.rs"],
moduleExists,
readModuleFile: () => "",
});
assert.equal(result.versions.duckdb, "0.1.1");
});
+144 -53
View File
@@ -207,6 +207,111 @@ jobs:
name: kingbase-native name: kingbase-native
path: "release-native/dbx-agent-kingbase-*" path: "release-native/dbx-agent-kingbase-*"
build-duckdb-native:
needs: [bump-versions]
strategy:
fail-fast: false
matrix:
include:
- runner: macos-latest
target: aarch64-apple-darwin
platform: macos-aarch64
extension: ""
- runner: macos-15-intel
target: x86_64-apple-darwin
platform: macos-x64
extension: ""
- runner: ubuntu-22.04-arm
target: aarch64-unknown-linux-gnu
platform: linux-aarch64
extension: ""
manylinux_image: quay.io/pypa/manylinux2014_aarch64
- runner: ubuntu-22.04
target: x86_64-unknown-linux-gnu
platform: linux-x64
extension: ""
manylinux_image: quay.io/pypa/manylinux2014_x86_64
- runner: windows-2022
target: aarch64-pc-windows-msvc
platform: windows-aarch64
extension: .exe
- runner: windows-2022
target: x86_64-win7-windows-msvc
platform: windows-x64
extension: .exe
win7: true
runs-on: ${{ matrix.runner }}
env:
CARGO_INCREMENTAL: "0"
CARGO_TARGET_DIR: ${{ github.workspace }}/target/duckdb-driver
RUSTC_WRAPPER: sccache
steps:
- uses: actions/checkout@v5
- uses: dtolnay/rust-toolchain@1.97.1
if: runner.os != 'Linux' && matrix.win7 != true
with:
targets: ${{ matrix.target }}
- name: Setup Rust for Windows 7
if: matrix.win7 == true
uses: dtolnay/rust-toolchain@nightly
with:
toolchain: nightly-2026-07-22
components: rust-src
- uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
with:
version: "v0.10.0"
- name: Build DuckDB native driver
shell: bash
run: |
if [ -n "${{ matrix.manylinux_image }}" ]; then
docker run --rm \
--user "$(id -u):$(id -g)" \
-e HOME=/tmp/dbx-rust-home \
-e CARGO_INCREMENTAL=0 \
-e CARGO_TARGET_DIR=/workspace/target/duckdb-driver \
-v "${{ github.workspace }}:/workspace" \
-w /workspace \
"${{ matrix.manylinux_image }}" \
bash -lc '
mkdir -p "$HOME"
curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.97.1
source "$HOME/.cargo/env"
cargo build \
--manifest-path agents/drivers/duckdb/Cargo.toml \
--locked \
--release \
--bin dbx-duckdb-driver \
--target "${{ matrix.target }}"
'
else
BUILD_STD=()
if [ "${{ matrix.win7 }}" = "true" ]; then
BUILD_STD=(-Z build-std=std,panic_abort)
fi
cargo build \
--manifest-path agents/drivers/duckdb/Cargo.toml \
--locked \
--release \
--bin dbx-duckdb-driver \
--target "${{ matrix.target }}" \
"${BUILD_STD[@]}"
fi
mkdir -p release-native
cp \
"target/duckdb-driver/${{ matrix.target }}/release/dbx-duckdb-driver${{ matrix.extension }}" \
"release-native/dbx-agent-duckdb-${{ matrix.platform }}${{ matrix.extension }}"
if [ -n "${{ matrix.manylinux_image }}" ]; then
docker run --rm \
--user "$(id -u):$(id -g)" \
-v "${{ github.workspace }}/release-native:/driver:ro" \
"${{ matrix.manylinux_image }}" \
"/driver/dbx-agent-duckdb-${{ matrix.platform }}" < /dev/null
fi
- uses: actions/upload-artifact@v4
with:
name: duckdb-native-${{ matrix.platform }}
path: "release-native/dbx-agent-duckdb-*"
build-jre: build-jre:
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
@@ -223,12 +328,14 @@ jobs:
- name: Build JRE for all platforms - name: Build JRE for all platforms
shell: bash shell: bash
run: | run: |
sudo apt-get update
sudo apt-get install -y zstd
MODULES="${{ matrix.modules }}" MODULES="${{ matrix.modules }}"
JLINK_OPTS="--strip-debug --no-header-files --no-man-pages --compress=zip-6" JLINK_OPTS="--strip-debug --no-header-files --no-man-pages --compress=zip-6"
JRE_KEY="${{ matrix.jre-key }}" JRE_KEY="${{ matrix.jre-key }}"
jlink --add-modules $MODULES $JLINK_OPTS --output dbx-jre jlink --add-modules $MODULES $JLINK_OPTS --output dbx-jre
tar czf dbx-jre-${JRE_KEY}-linux-x64.tar.gz dbx-jre tar cf - dbx-jre | zstd -q -19 -T0 -o dbx-jre-${JRE_KEY}-linux-x64.tar.zst
rm -rf dbx-jre rm -rf dbx-jre
ADOPTIUM="https://api.adoptium.net/v3/binary/latest/${{ matrix.java-version }}/ga" ADOPTIUM="https://api.adoptium.net/v3/binary/latest/${{ matrix.java-version }}/ga"
@@ -269,18 +376,18 @@ jobs:
CROSS_OPTS="--disable-plugin strip-native-debug-symbols" CROSS_OPTS="--disable-plugin strip-native-debug-symbols"
fi fi
jlink --module-path "$JMODS" --add-modules $MODULES $JLINK_OPTS $CROSS_OPTS --output dbx-jre jlink --module-path "$JMODS" --add-modules $MODULES $JLINK_OPTS $CROSS_OPTS --output dbx-jre
tar czf dbx-jre-${JRE_KEY}-$platform.tar.gz dbx-jre tar cf - dbx-jre | zstd -q -19 -T0 -o dbx-jre-${JRE_KEY}-$platform.tar.zst
rm -rf dbx-jre jdk-$platform rm -rf dbx-jre jdk-$platform
done done
ls -lh dbx-jre-*.tar.gz ls -lh dbx-jre-*.tar.zst
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
with: with:
name: jre-${{ matrix.jre-key }} name: jre-${{ matrix.jre-key }}
path: "dbx-jre-*.tar.gz" path: "dbx-jre-*.tar.zst"
release: release:
needs: [bump-versions, build-agents, build-oracle-native, build-xugu-native, build-kingbase-native, build-jre] needs: [bump-versions, build-agents, build-oracle-native, build-xugu-native, build-kingbase-native, build-duckdb-native, build-jre]
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Create DBX bot release token - name: Create DBX bot release token
@@ -310,7 +417,8 @@ jobs:
find artifacts/oracle-native -type f -name 'dbx-agent-oracle-*' -exec cp {} release/ \; find artifacts/oracle-native -type f -name 'dbx-agent-oracle-*' -exec cp {} release/ \;
find artifacts/xugu-native -type f -name 'dbx-agent-xugu-*' -exec cp {} release/ \; find artifacts/xugu-native -type f -name 'dbx-agent-xugu-*' -exec cp {} release/ \;
find artifacts/kingbase-native -type f -name 'dbx-agent-kingbase-*' -exec cp {} release/ \; find artifacts/kingbase-native -type f -name 'dbx-agent-kingbase-*' -exec cp {} release/ \;
find artifacts -name 'dbx-jre-*.tar.gz' -exec cp {} release/ \; find artifacts/duckdb-native-* -type f -name 'dbx-agent-duckdb-*' -exec cp {} release/ \;
find artifacts -name 'dbx-jre-*.tar.zst' -exec cp {} release/ \;
ls -lh release/ ls -lh release/
- name: Add versions to agent artifact filenames - name: Add versions to agent artifact filenames
@@ -325,6 +433,7 @@ jobs:
TAG="${GITHUB_REF_NAME}" TAG="${GITHUB_REF_NAME}"
REPO="${GITHUB_REPOSITORY}" REPO="${GITHUB_REPOSITORY}"
RELEASE_VERSION="${TAG#agents-v}" RELEASE_VERSION="${TAG#agents-v}"
CURRENT_APP_VERSION=$(awk -F'"' '/^version = "/ { print $2; exit }' src-tauri/Cargo.toml)
get_module_version() { get_module_version() {
local name="$1" local name="$1"
@@ -349,7 +458,7 @@ jobs:
"${name}": { "${name}": {
"version": "${version}", "version": "${version}",
"label": "${label}", "label": "${label}",
"min_app_version": "0.6.0", "min_app_version": "${CURRENT_APP_VERSION}",
"jre": "${jre_key}", "jre": "${jre_key}",
"external_driver_required": ${external_driver}, "external_driver_required": ${external_driver},
${native_line} ${native_line}
@@ -359,13 +468,13 @@ jobs:
} }
generate_native_entry() { generate_native_entry() {
local name="$1" label="$2" jre_key="$3" version="$4" native_json="$5" local name="$1" label="$2" jre_key="$3" version="$4" native_json="$5" min_app_version="$6"
local legacy_jar_url="https://github.com/${REPO}/releases/download/${TAG}/dbx-agent-${name}-legacy-placeholder.jar" local legacy_jar_url="https://github.com/${REPO}/releases/download/${TAG}/dbx-agent-${name}-legacy-placeholder.jar"
cat <<ENTRY cat <<ENTRY
"${name}": { "${name}": {
"version": "${version}", "version": "${version}",
"label": "${label}", "label": "${label}",
"min_app_version": "0.6.0", "min_app_version": "${min_app_version}",
"jre": "${jre_key}", "jre": "${jre_key}",
"external_driver_required": false, "external_driver_required": false,
"native": {${native_json} "native": {${native_json}
@@ -379,22 +488,27 @@ jobs:
local name="$1" local name="$1"
case "$name" in case "$name" in
kingbase) echo "人大金仓 KingbaseES" ;; kingbase) echo "人大金仓 KingbaseES" ;;
duckdb) echo "DuckDB" ;;
xugu) echo "虚谷 XuguDB" ;; xugu) echo "虚谷 XuguDB" ;;
*) echo "$name" ;; *) echo "$name" ;;
esac esac
} }
native_only_min_app_version() {
echo "$CURRENT_APP_VERSION"
}
generate_jre_platforms() { generate_jre_platforms() {
local jre_key="$1" local jre_key="$1"
local PLATFORMS="" local PLATFORMS=""
for f in release/dbx-jre-${jre_key}-*.tar.gz; do for f in release/dbx-jre-${jre_key}-*.tar.zst; do
[ ! -f "$f" ] && continue [ ! -f "$f" ] && continue
p=$(basename "$f" .tar.gz | sed "s/dbx-jre-${jre_key}-//") p=$(basename "$f" .tar.zst | sed "s/dbx-jre-${jre_key}-//")
sha256=$(sha256sum "$f" | cut -d' ' -f1) sha256=$(sha256sum "$f" | cut -d' ' -f1)
size=$(stat -c%s "$f") size=$(stat -c%s "$f")
url="https://github.com/${REPO}/releases/download/${TAG}/$(basename $f)" url="https://github.com/${REPO}/releases/download/${TAG}/$(basename $f)"
[ -n "$PLATFORMS" ] && PLATFORMS="${PLATFORMS},"$'\n' [ -n "$PLATFORMS" ] && PLATFORMS="${PLATFORMS},"$'\n'
PLATFORMS="${PLATFORMS} \"${p}\": { \"url\": \"${url}\", \"sha256\": \"${sha256}\", \"size\": ${size} }" PLATFORMS="${PLATFORMS} \"${p}\": { \"url\": \"${url}\", \"sha256\": \"${sha256}\", \"size\": ${size}, \"format\": \"tar_zstd\" }"
done done
echo "$PLATFORMS" echo "$PLATFORMS"
} }
@@ -439,15 +553,16 @@ jobs:
[ -n "$DRIVERS" ] && DRIVERS="${DRIVERS},"$'\n' [ -n "$DRIVERS" ] && DRIVERS="${DRIVERS},"$'\n'
DRIVERS="${DRIVERS}$(generate_jar_entry "$name" "$label" "$f" "$jre_key" "$version" "$external_driver" "$native_json")" DRIVERS="${DRIVERS}$(generate_jar_entry "$name" "$label" "$f" "$jre_key" "$version" "$external_driver" "$native_json")"
done done
for name in oracle xugu kingbase; do for name in oracle xugu kingbase duckdb; do
version=$(get_module_version "$name") version=$(get_module_version "$name")
[ -f "release/dbx-agent-${name}-${version}.jar" ] && continue [ -f "release/dbx-agent-${name}-${version}.jar" ] && continue
native_json=$(generate_native_platforms "$name" "$version") native_json=$(generate_native_platforms "$name" "$version")
[ -z "$native_json" ] && continue [ -z "$native_json" ] && continue
label=$(native_only_label "$name") label=$(native_only_label "$name")
jre_key=$(detect_jre_key "$name") jre_key=$(detect_jre_key "$name")
min_app_version=$(native_only_min_app_version "$name")
[ -n "$DRIVERS" ] && DRIVERS="${DRIVERS},"$'\n' [ -n "$DRIVERS" ] && DRIVERS="${DRIVERS},"$'\n'
DRIVERS="${DRIVERS}$(generate_native_entry "$name" "$label" "$jre_key" "$version" "$native_json")" DRIVERS="${DRIVERS}$(generate_native_entry "$name" "$label" "$jre_key" "$version" "$native_json" "$min_app_version")"
done done
cat > release/agent-registry.json <<EOF cat > release/agent-registry.json <<EOF
@@ -470,12 +585,18 @@ jobs:
echo "=== agent-registry.json ===" echo "=== agent-registry.json ==="
cat release/agent-registry.json cat release/agent-registry.json
- name: Build single-driver ZIP bundles
run: python3 agents/scripts/build_driver_zips.py release
- name: Build offline ZIP bundles - name: Build offline ZIP bundles
run: bash agents/scripts/build_offline_zip.sh release run: bash agents/scripts/build_offline_zip.sh release
- name: Build single-driver packages
run: |
sudo apt-get update
sudo apt-get install -y zstd
python3 agents/scripts/build_driver_zips.py release --cleanup-sources
python3 -m json.tool release/agent-registry.json > /dev/null
echo "=== final agent-registry.json ==="
cat release/agent-registry.json
- name: Generate release notes - name: Generate release notes
env: env:
MODULE_VERSIONS: ${{ needs.bump-versions.outputs.versions }} MODULE_VERSIONS: ${{ needs.bump-versions.outputs.versions }}
@@ -489,17 +610,14 @@ jobs:
} }
for name in $(module_names); do for name in $(module_names); do
version=$(echo "$MODULE_VERSIONS" | python3 -c "import sys,json; print(json.load(sys.stdin).get('$name',''))") version=$(echo "$MODULE_VERSIONS" | python3 -c "import sys,json; print(json.load(sys.stdin).get('$name',''))")
[ -f "release/dbx-agent-${name}-${version}.jar" ] && DRIVER_NAMES+=("$name") compgen -G "release/dbx-agent-${name}-${version}*.tar.zst" > /dev/null && DRIVER_NAMES+=("$name")
done
for name in oracle xugu kingbase; do
[ -f "release/dbx-agent-${name}.jar" ] && continue
compgen -G "release/dbx-agent-${name}-*" > /dev/null && DRIVER_NAMES+=("$name")
done done
native_only_label() { native_only_label() {
local name="$1" local name="$1"
case "$name" in case "$name" in
kingbase) echo "人大金仓 KingbaseES" ;; kingbase) echo "人大金仓 KingbaseES" ;;
duckdb) echo "DuckDB" ;;
oracle) echo "Oracle" ;; oracle) echo "Oracle" ;;
xugu) echo "虚谷 XuguDB" ;; xugu) echo "虚谷 XuguDB" ;;
*) echo "$name" ;; *) echo "$name" ;;
@@ -588,47 +706,20 @@ jobs:
- name: Upload to R2 - name: Upload to R2
env: env:
MODULE_VERSIONS: ${{ needs.bump-versions.outputs.versions }}
PREV_VERSIONS: ${{ needs.bump-versions.outputs.prev_versions }}
AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
R2_ENDPOINT: "https://${{ secrets.R2_ACCOUNT_ID }}.r2.cloudflarestorage.com" R2_ENDPOINT: "https://${{ secrets.R2_ACCOUNT_ID }}.r2.cloudflarestorage.com"
R2_BUCKET: "s3://${{ secrets.R2_BUCKET_NAME }}" R2_BUCKET: "s3://${{ secrets.R2_BUCKET_NAME }}"
run: | run: |
# Always upload agent-registry.json # Always upload the current registry.
aws s3 cp release/agent-registry.json \ aws s3 cp release/agent-registry.json \
"$R2_BUCKET/agents/agent-registry.json" \ "$R2_BUCKET/agents/agent-registry.json" \
--endpoint-url "$R2_ENDPOINT" --endpoint-url "$R2_ENDPOINT"
echo "Uploaded agent-registry.json" echo "Uploaded agent-registry.json"
# Upload only changed driver JARs # Upload all single-driver packages.
UPLOADED=0 for f in release/dbx-agent-*.tar.zst; do
SKIPPED=0
module_names() {
echo "$MODULE_VERSIONS" | python3 -c 'import sys,json; print("\n".join(sorted(json.load(sys.stdin))))'
}
for name in $(module_names); do
old_ver=$(echo "$PREV_VERSIONS" | python3 -c "import sys,json; print(json.load(sys.stdin).get('$name',''))" 2>/dev/null)
new_ver=$(echo "$MODULE_VERSIONS" | python3 -c "import sys,json; print(json.load(sys.stdin).get('$name',''))" 2>/dev/null)
f="release/dbx-agent-${name}-${new_ver}.jar"
[ ! -f "$f" ] && continue
if [ "$old_ver" = "$new_ver" ] && [ -n "$old_ver" ]; then
echo "Skip $(basename $f) (unchanged $old_ver)"
SKIPPED=$((SKIPPED + 1))
continue
fi
aws s3 cp "$f" \
"$R2_BUCKET/agents/drivers/$(basename $f)" \
--endpoint-url "$R2_ENDPOINT"
echo "Uploaded $(basename $f) ($old_ver → $new_ver)"
UPLOADED=$((UPLOADED + 1))
done
echo "Drivers: $UPLOADED uploaded, $SKIPPED skipped"
# Upload versioned native binaries and all single-driver ZIP packages.
for f in release/dbx-agent-*; do
[ -f "$f" ] || continue [ -f "$f" ] || continue
[[ "$f" != *.jar ]] || continue
aws s3 cp "$f" \ aws s3 cp "$f" \
"$R2_BUCKET/agents/drivers/$(basename $f)" \ "$R2_BUCKET/agents/drivers/$(basename $f)" \
--endpoint-url "$R2_ENDPOINT" --endpoint-url "$R2_ENDPOINT"
@@ -636,7 +727,7 @@ jobs:
done done
# Upload JRE — always overwrite to ensure latest modules # Upload JRE — always overwrite to ensure latest modules
for f in release/dbx-jre-*.tar.gz; do for f in release/dbx-jre-*.tar.zst; do
[ ! -f "$f" ] && continue [ ! -f "$f" ] && continue
key="agents/jre/$(basename $f)" key="agents/jre/$(basename $f)"
aws s3 cp "$f" "$R2_BUCKET/$key" --endpoint-url "$R2_ENDPOINT" aws s3 cp "$f" "$R2_BUCKET/$key" --endpoint-url "$R2_ENDPOINT"
+29 -14
View File
@@ -141,9 +141,10 @@ jobs:
# sccache cannot reuse Cargo incremental artifacts, so avoid generating them in CI. # sccache cannot reuse Cargo incremental artifacts, so avoid generating them in CI.
CARGO_INCREMENTAL: "0" CARGO_INCREMENTAL: "0"
RUSTC_WRAPPER: sccache RUSTC_WRAPPER: sccache
# The fast lane skips only bundled DuckDB while retaining the other default capabilities. # The fast lane skips only system font discovery while retaining the other default capabilities.
RUST_FEATURE_MODE: ${{ github.event_name == 'pull_request' && needs.changes.outputs.rust_full != 'true' && 'fast' || 'full' }} RUST_FEATURE_MODE: ${{ github.event_name == 'pull_request' && needs.changes.outputs.rust_full != 'true' && 'fast' || 'full' }}
RUST_FAST_FEATURES: dbx/mq-admin,dbx/sqlite-sqlcipher,dbx-core/mq-admin,dbx-core/sqlite-sqlcipher,dbx-web/mq-admin,dbx-web/sqlite-sqlcipher RUST_FAST_FEATURES: dbx/duckdb-sidecar,dbx/mq-admin,dbx/sqlite-sqlcipher,dbx-core/duckdb-sidecar,dbx-core/mq-admin,dbx-core/sqlite-sqlcipher,dbx-web/duckdb-sidecar,dbx-web/mq-admin,dbx-web/sqlite-sqlcipher
RUST_FULL_FEATURES: dbx/duckdb-sidecar,dbx/mq-admin,dbx/sqlite-sqlcipher,dbx/system-fonts,dbx-core/duckdb-sidecar,dbx-core/mq-admin,dbx-core/sqlite-sqlcipher,dbx-core/system-fonts,dbx-web/duckdb-sidecar,dbx-web/mq-admin,dbx-web/sqlite-sqlcipher,dbx-web/system-fonts
# Fork PRs cannot read repository secrets, so retain the GHA backend for them. # Fork PRs cannot read repository secrets, so retain the GHA backend for them.
SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }} SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }}
steps: steps:
@@ -203,7 +204,7 @@ jobs:
if [ "$RUST_FEATURE_MODE" = "fast" ]; then if [ "$RUST_FEATURE_MODE" = "fast" ]; then
cargo clippy --workspace --locked --all-targets --no-default-features --features "$RUST_FAST_FEATURES" -- -D warnings cargo clippy --workspace --locked --all-targets --no-default-features --features "$RUST_FAST_FEATURES" -- -D warnings
else else
cargo clippy --workspace --locked --all-targets -- -D warnings cargo clippy --workspace --locked --all-targets --no-default-features --features "$RUST_FULL_FEATURES" -- -D warnings
fi fi
- name: Show sccache stats - name: Show sccache stats
@@ -219,9 +220,10 @@ jobs:
# sccache cannot reuse Cargo incremental artifacts, so avoid generating them in CI. # sccache cannot reuse Cargo incremental artifacts, so avoid generating them in CI.
CARGO_INCREMENTAL: "0" CARGO_INCREMENTAL: "0"
RUSTC_WRAPPER: sccache RUSTC_WRAPPER: sccache
# The fast lane skips only bundled DuckDB while retaining the other default capabilities. # The fast lane skips only system font discovery while retaining the other default capabilities.
RUST_FEATURE_MODE: ${{ github.event_name == 'pull_request' && needs.changes.outputs.rust_full != 'true' && 'fast' || 'full' }} RUST_FEATURE_MODE: ${{ github.event_name == 'pull_request' && needs.changes.outputs.rust_full != 'true' && 'fast' || 'full' }}
RUST_FAST_FEATURES: dbx/mq-admin,dbx/sqlite-sqlcipher,dbx-core/mq-admin,dbx-core/sqlite-sqlcipher,dbx-web/mq-admin,dbx-web/sqlite-sqlcipher RUST_FAST_FEATURES: dbx/duckdb-sidecar,dbx/mq-admin,dbx/sqlite-sqlcipher,dbx-core/duckdb-sidecar,dbx-core/mq-admin,dbx-core/sqlite-sqlcipher,dbx-web/duckdb-sidecar,dbx-web/mq-admin,dbx-web/sqlite-sqlcipher
RUST_FULL_FEATURES: dbx/duckdb-sidecar,dbx/mq-admin,dbx/sqlite-sqlcipher,dbx/system-fonts,dbx-core/duckdb-sidecar,dbx-core/mq-admin,dbx-core/sqlite-sqlcipher,dbx-core/system-fonts,dbx-web/duckdb-sidecar,dbx-web/mq-admin,dbx-web/sqlite-sqlcipher,dbx-web/system-fonts
# Fork PRs cannot read repository secrets, so retain the GHA backend for them. # Fork PRs cannot read repository secrets, so retain the GHA backend for them.
SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }} SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }}
steps: steps:
@@ -278,7 +280,7 @@ jobs:
if [ "$RUST_FEATURE_MODE" = "fast" ]; then if [ "$RUST_FEATURE_MODE" = "fast" ]; then
cargo test --workspace --locked --no-default-features --features "$RUST_FAST_FEATURES" cargo test --workspace --locked --no-default-features --features "$RUST_FAST_FEATURES"
else else
cargo test --workspace --locked cargo test --workspace --locked --no-default-features --features "$RUST_FULL_FEATURES"
fi fi
- name: Show sccache stats - name: Show sccache stats
@@ -395,6 +397,12 @@ jobs:
- 'plugins/jdbc/**' - 'plugins/jdbc/**'
agents: agents:
- 'agents/**' - 'agents/**'
- 'crates/dbx-core/Cargo.toml'
- 'Cargo.toml'
- 'Cargo.lock'
- '.github/scripts/bump-agent-versions.mjs'
- '.github/scripts/bump-agent-versions.test.mjs'
- '.github/workflows/agents-release.yml'
nix: nix:
# This fast check validates only the fixed-output pnpm dependency closure. # This fast check validates only the fixed-output pnpm dependency closure.
- 'package.json' - 'package.json'
@@ -441,13 +449,6 @@ jobs:
;; ;;
esac esac
# Feature-gated DuckDB code must keep the full default-feature checks on its PR.
if echo "$file" | grep -Eqi 'duckdb' \
|| { [ -f "$file" ] && grep -Eqi 'duckdb-bundled|cfg[^[:cntrl:]]*duckdb|feature[^[:cntrl:]]*duckdb' "$file"; } \
|| git diff "$BASE_SHA" HEAD -- "$file" | grep -Eqi 'duckdb-bundled|cfg[^[:cntrl:]]*duckdb|feature[^[:cntrl:]]*duckdb'; then
full=true
break
fi
done < <(git diff --name-only "$BASE_SHA" HEAD -- Cargo.toml Cargo.lock 'rust-toolchain*' crates src-tauri .github/workflows/ci.yml) done < <(git diff --name-only "$BASE_SHA" HEAD -- Cargo.toml Cargo.lock 'rust-toolchain*' crates src-tauri .github/workflows/ci.yml)
fi fi
@@ -476,11 +477,25 @@ jobs:
with: with:
go-version: "1.22.x" go-version: "1.22.x"
- name: Setup Rust
uses: dtolnay/rust-toolchain@1.97.1
- name: Setup Gradle - name: Setup Gradle
uses: gradle/actions/setup-gradle@v4 uses: gradle/actions/setup-gradle@v4
- name: Install packaging tools
run: |
sudo apt-get update
sudo apt-get install -y zstd
- name: Agent script tests - name: Agent script tests
run: python3 -m unittest discover -s scripts -p '*_test.py' run: |
python3 -m unittest discover -s scripts -p '*_test.py'
node --test ../.github/scripts/bump-agent-versions.test.mjs
- name: DuckDB driver tests
working-directory: ..
run: cargo test --manifest-path agents/drivers/duckdb/Cargo.toml --locked
- name: Agent validation - name: Agent validation
run: python3 scripts/validate_agents.py run: python3 scripts/validate_agents.py
+1 -1
View File
@@ -503,7 +503,7 @@ jobs:
RUSTFLAGS: ${{ matrix.rustflags }} RUSTFLAGS: ${{ matrix.rustflags }}
run: | run: |
cargo zigbuild --release -p dbx-web --target ${{ matrix.target }} \ cargo zigbuild --release -p dbx-web --target ${{ matrix.target }} \
--no-default-features --features "duckdb-bundled,mq-admin" --no-default-features --features "duckdb-sidecar,mq-admin"
- name: Package static browser runtime - name: Package static browser runtime
env: env:
Generated
+19 -431
View File
@@ -96,7 +96,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"const-random",
"getrandom 0.3.4", "getrandom 0.3.4",
"once_cell", "once_cell",
"version_check", "version_check",
@@ -231,169 +230,6 @@ version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
[[package]]
name = "arrow"
version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3"
dependencies = [
"arrow-arith",
"arrow-array",
"arrow-buffer",
"arrow-cast",
"arrow-data",
"arrow-ord",
"arrow-row",
"arrow-schema",
"arrow-select",
"arrow-string",
]
[[package]]
name = "arrow-arith"
version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"chrono",
"num-traits",
]
[[package]]
name = "arrow-array"
version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69"
dependencies = [
"ahash 0.8.12",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"chrono",
"half",
"hashbrown 0.17.1",
"num-complex",
"num-integer",
"num-traits",
]
[[package]]
name = "arrow-buffer"
version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5"
dependencies = [
"bytes",
"half",
"num-bigint",
"num-traits",
]
[[package]]
name = "arrow-cast"
version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-ord",
"arrow-schema",
"arrow-select",
"atoi",
"base64 0.22.1",
"chrono",
"comfy-table",
"half",
"lexical-core",
"num-traits",
"ryu",
]
[[package]]
name = "arrow-data"
version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b"
dependencies = [
"arrow-buffer",
"arrow-schema",
"half",
"num-integer",
"num-traits",
]
[[package]]
name = "arrow-ord"
version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
]
[[package]]
name = "arrow-row"
version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"half",
]
[[package]]
name = "arrow-schema"
version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e"
dependencies = [
"bitflags 2.13.1",
]
[[package]]
name = "arrow-select"
version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad"
dependencies = [
"ahash 0.8.12",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"num-traits",
]
[[package]]
name = "arrow-string"
version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
"memchr",
"num-traits",
"regex",
"regex-syntax",
]
[[package]] [[package]]
name = "async-broadcast" name = "async-broadcast"
version = "0.7.2" version = "0.7.2"
@@ -457,7 +293,7 @@ dependencies = [
"futures-lite", "futures-lite",
"parking", "parking",
"polling", "polling",
"rustix 1.1.4", "rustix",
"slab", "slab",
"windows-sys 0.61.2", "windows-sys 0.61.2",
] ]
@@ -488,7 +324,7 @@ dependencies = [
"cfg-if", "cfg-if",
"event-listener", "event-listener",
"futures-lite", "futures-lite",
"rustix 1.1.4", "rustix",
] ]
[[package]] [[package]]
@@ -514,7 +350,7 @@ dependencies = [
"cfg-if", "cfg-if",
"futures-core", "futures-core",
"futures-io", "futures-io",
"rustix 1.1.4", "rustix",
"signal-hook-registry", "signal-hook-registry",
"slab", "slab",
"windows-sys 0.61.2", "windows-sys 0.61.2",
@@ -595,15 +431,6 @@ dependencies = [
"system-deps", "system-deps",
] ]
[[package]]
name = "atoi"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528"
dependencies = [
"num-traits",
]
[[package]] [[package]]
name = "atoi_simd" name = "atoi_simd"
version = "0.16.1" version = "0.16.1"
@@ -1065,7 +892,7 @@ dependencies = [
"log", "log",
"quick-xml 0.37.5", "quick-xml 0.37.5",
"serde", "serde",
"zip 4.6.1", "zip",
] ]
[[package]] [[package]]
@@ -1110,12 +937,6 @@ dependencies = [
"toml 0.9.12+spec-1.1.0", "toml 0.9.12+spec-1.1.0",
] ]
[[package]]
name = "cast"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]] [[package]]
name = "cbc" name = "cbc"
version = "0.1.2" version = "0.1.2"
@@ -1289,17 +1110,6 @@ dependencies = [
"tokio-util", "tokio-util",
] ]
[[package]]
name = "comfy-table"
version = "7.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a"
dependencies = [
"crossterm",
"unicode-segmentation",
"unicode-width",
]
[[package]] [[package]]
name = "compression-codecs" name = "compression-codecs"
version = "0.4.38" version = "0.4.38"
@@ -1574,28 +1384,6 @@ version = "0.8.22"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
[[package]]
name = "crossterm"
version = "0.28.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6"
dependencies = [
"bitflags 2.13.1",
"crossterm_winapi",
"parking_lot",
"rustix 0.38.44",
"winapi",
]
[[package]]
name = "crossterm_winapi"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b"
dependencies = [
"winapi",
]
[[package]] [[package]]
name = "crunchy" name = "crunchy"
version = "0.2.4" version = "0.2.4"
@@ -1843,7 +1631,6 @@ dependencies = [
"csv", "csv",
"dbx-core", "dbx-core",
"deadpool-postgres", "deadpool-postgres",
"duckdb",
"font-kit", "font-kit",
"futures", "futures",
"libloading 0.8.9", "libloading 0.8.9",
@@ -1882,7 +1669,7 @@ dependencies = [
"tokio-postgres", "tokio-postgres",
"tokio-util", "tokio-util",
"uuid", "uuid",
"zip 4.6.1", "zip",
] ]
[[package]] [[package]]
@@ -1915,7 +1702,6 @@ dependencies = [
"chrono", "chrono",
"csv", "csv",
"deadpool-postgres", "deadpool-postgres",
"duckdb",
"encoding_rs", "encoding_rs",
"flate2", "flate2",
"font-kit", "font-kit",
@@ -1966,7 +1752,8 @@ dependencies = [
"uuid", "uuid",
"webpki-roots 0.26.11", "webpki-roots 0.26.11",
"winnow 1.0.4", "winnow 1.0.4",
"zip 4.6.1", "zip",
"zstd",
] ]
[[package]] [[package]]
@@ -2341,23 +2128,6 @@ version = "0.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5"
[[package]]
name = "duckdb"
version = "1.10505.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "970e05eedd3f55c435194d9104f90a9b4a79a80d6e73251bc9ff43e178130c4e"
dependencies = [
"arrow",
"cast",
"comfy-table",
"fallible-iterator 0.3.0",
"fallible-streaming-iterator",
"hashlink 0.10.0",
"libduckdb-sys",
"num-integer",
"strum",
]
[[package]] [[package]]
name = "dunce" name = "dunce"
version = "1.0.5" version = "1.0.5"
@@ -3055,7 +2825,7 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8"
dependencies = [ dependencies = [
"rustix 1.1.4", "rustix",
"windows-link 0.2.1", "windows-link 0.2.1",
] ]
@@ -3307,7 +3077,6 @@ checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"crunchy", "crunchy",
"num-traits",
"zerocopy", "zerocopy",
] ]
@@ -3369,15 +3138,6 @@ dependencies = [
"hashbrown 0.14.5", "hashbrown 0.14.5",
] ]
[[package]]
name = "hashlink"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
dependencies = [
"hashbrown 0.15.5",
]
[[package]] [[package]]
name = "heck" name = "heck"
version = "0.4.1" version = "0.4.1"
@@ -4242,63 +4002,6 @@ dependencies = [
"spin", "spin",
] ]
[[package]]
name = "lexical-core"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594"
dependencies = [
"lexical-parse-float",
"lexical-parse-integer",
"lexical-util",
"lexical-write-float",
"lexical-write-integer",
]
[[package]]
name = "lexical-parse-float"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56"
dependencies = [
"lexical-parse-integer",
"lexical-util",
]
[[package]]
name = "lexical-parse-integer"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34"
dependencies = [
"lexical-util",
]
[[package]]
name = "lexical-util"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17"
[[package]]
name = "lexical-write-float"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361"
dependencies = [
"lexical-util",
"lexical-write-integer",
]
[[package]]
name = "lexical-write-integer"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df"
dependencies = [
"lexical-util",
]
[[package]] [[package]]
name = "libappindicator" name = "libappindicator"
version = "0.9.0" version = "0.9.0"
@@ -4344,23 +4047,6 @@ dependencies = [
"pkg-config", "pkg-config",
] ]
[[package]]
name = "libduckdb-sys"
version = "1.10505.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6cb514dab5e271e849235c1cb98bd65a2ae107fbd619a6740219319c54a71d95"
dependencies = [
"cc",
"flate2",
"pkg-config",
"serde",
"serde_json",
"tar",
"ureq",
"vcpkg",
"zip 6.0.0",
]
[[package]] [[package]]
name = "libloading" name = "libloading"
version = "0.7.4" version = "0.7.4"
@@ -4428,12 +4114,6 @@ dependencies = [
"vcpkg", "vcpkg",
] ]
[[package]]
name = "linux-raw-sys"
version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
[[package]] [[package]]
name = "linux-raw-sys" name = "linux-raw-sys"
version = "0.12.1" version = "0.12.1"
@@ -5059,15 +4739,6 @@ dependencies = [
"smallvec", "smallvec",
] ]
[[package]]
name = "num-complex"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
dependencies = [
"num-traits",
]
[[package]] [[package]]
name = "num-conv" name = "num-conv"
version = "0.2.2" version = "0.2.2"
@@ -5100,7 +4771,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [ dependencies = [
"autocfg", "autocfg",
"libm",
] ]
[[package]] [[package]]
@@ -5929,7 +5599,7 @@ dependencies = [
"concurrent-queue", "concurrent-queue",
"hermit-abi", "hermit-abi",
"pin-project-lite", "pin-project-lite",
"rustix 1.1.4", "rustix",
"windows-sys 0.61.2", "windows-sys 0.61.2",
] ]
@@ -6791,7 +6461,7 @@ dependencies = [
"bitflags 2.13.1", "bitflags 2.13.1",
"fallible-iterator 0.3.0", "fallible-iterator 0.3.0",
"fallible-streaming-iterator", "fallible-streaming-iterator",
"hashlink 0.9.1", "hashlink",
"libsqlite3-sys", "libsqlite3-sys",
"smallvec", "smallvec",
] ]
@@ -6976,19 +6646,6 @@ dependencies = [
"subtle", "subtle",
] ]
[[package]]
name = "rustix"
version = "0.38.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
dependencies = [
"bitflags 2.13.1",
"errno",
"libc",
"linux-raw-sys 0.4.15",
"windows-sys 0.59.0",
]
[[package]] [[package]]
name = "rustix" name = "rustix"
version = "1.1.4" version = "1.1.4"
@@ -6998,7 +6655,7 @@ dependencies = [
"bitflags 2.13.1", "bitflags 2.13.1",
"errno", "errno",
"libc", "libc",
"linux-raw-sys 0.12.1", "linux-raw-sys",
"windows-sys 0.61.2", "windows-sys 0.61.2",
] ]
@@ -7973,27 +7630,6 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "strum"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"
dependencies = [
"strum_macros",
]
[[package]]
name = "strum_macros"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
dependencies = [
"heck 0.5.0",
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]] [[package]]
name = "subtle" name = "subtle"
version = "2.6.1" version = "2.6.1"
@@ -8512,7 +8148,7 @@ dependencies = [
"tokio", "tokio",
"url", "url",
"windows-sys 0.60.2", "windows-sys 0.60.2",
"zip 4.6.1", "zip",
] ]
[[package]] [[package]]
@@ -8639,7 +8275,7 @@ dependencies = [
"fastrand", "fastrand",
"getrandom 0.4.3", "getrandom 0.4.3",
"once_cell", "once_cell",
"rustix 1.1.4", "rustix",
"windows-sys 0.61.2", "windows-sys 0.61.2",
] ]
@@ -9457,34 +9093,6 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "ureq"
version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0"
dependencies = [
"base64 0.22.1",
"log",
"percent-encoding",
"rustls 0.23.42",
"rustls-pki-types",
"ureq-proto",
"utf8-zero",
"webpki-roots 1.0.9",
]
[[package]]
name = "ureq-proto"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c"
dependencies = [
"base64 0.22.1",
"http",
"httparse",
"log",
]
[[package]] [[package]]
name = "url" name = "url"
version = "2.5.8" version = "2.5.8"
@@ -9510,12 +9118,6 @@ dependencies = [
"url", "url",
] ]
[[package]]
name = "utf8-zero"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
[[package]] [[package]]
name = "utf8_iter" name = "utf8_iter"
version = "1.0.4" version = "1.0.4"
@@ -9744,7 +9346,7 @@ checksum = "016ccf01d1c58b6f8999612813e17c9b2390f7d70671428869913310f83f54b8"
dependencies = [ dependencies = [
"cc", "cc",
"downcast-rs", "downcast-rs",
"rustix 1.1.4", "rustix",
"smallvec", "smallvec",
"wayland-sys", "wayland-sys",
] ]
@@ -9756,7 +9358,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3c36a0f861ad76d0901f2800b46321410d9f73f2ea88aac0650d86c32688073" checksum = "e3c36a0f861ad76d0901f2800b46321410d9f73f2ea88aac0650d86c32688073"
dependencies = [ dependencies = [
"bitflags 2.13.1", "bitflags 2.13.1",
"rustix 1.1.4", "rustix",
"wayland-backend", "wayland-backend",
"wayland-scanner", "wayland-scanner",
] ]
@@ -10588,7 +10190,7 @@ dependencies = [
"libc", "libc",
"log", "log",
"os_pipe", "os_pipe",
"rustix 1.1.4", "rustix",
"thiserror 2.0.19", "thiserror 2.0.19",
"tree_magic_mini", "tree_magic_mini",
"wayland-backend", "wayland-backend",
@@ -10684,7 +10286,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414"
dependencies = [ dependencies = [
"gethostname", "gethostname",
"rustix 1.1.4", "rustix",
"x11rb-protocol", "x11rb-protocol",
] ]
@@ -10713,7 +10315,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
dependencies = [ dependencies = [
"libc", "libc",
"rustix 1.1.4", "rustix",
] ]
[[package]] [[package]]
@@ -10772,7 +10374,7 @@ dependencies = [
"hex", "hex",
"libc", "libc",
"ordered-stream", "ordered-stream",
"rustix 1.1.4", "rustix",
"serde", "serde",
"serde_repr", "serde_repr",
"tracing", "tracing",
@@ -10932,20 +10534,6 @@ dependencies = [
"zstd", "zstd",
] ]
[[package]]
name = "zip"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b"
dependencies = [
"arbitrary",
"crc32fast",
"flate2",
"indexmap 2.14.0",
"memchr",
"zopfli",
]
[[package]] [[package]]
name = "zlib-rs" name = "zlib-rs"
version = "0.6.6" version = "0.6.6"
+2 -2
View File
@@ -25,7 +25,7 @@ help:
@printf '%s\n' 'App:' @printf '%s\n' 'App:'
@printf ' %-23s %s\n' 'make' 'Start the local desktop development environment' @printf ' %-23s %s\n' 'make' 'Start the local desktop development environment'
@printf ' %-23s %s\n' 'make dev' 'Start the local desktop development environment' @printf ' %-23s %s\n' 'make dev' 'Start the local desktop development environment'
@printf ' %-23s %s\n' 'make dev-fast' 'Start Tauri dev without default Rust features' @printf ' %-23s %s\n' 'make dev-fast' 'Start lightweight Tauri dev with DuckDB sidecar support'
@printf ' %-23s %s\n' 'make dev-web' 'Start the web frontend development server' @printf ' %-23s %s\n' 'make dev-web' 'Start the web frontend development server'
@printf ' %-23s %s\n' 'make dev-backend' 'Start the web backend development server' @printf ' %-23s %s\n' 'make dev-backend' 'Start the web backend development server'
@printf ' %-23s %s\n' 'make build' 'Run type checks and build the desktop frontend' @printf ' %-23s %s\n' 'make build' 'Run type checks and build the desktop frontend'
@@ -75,7 +75,7 @@ dev: node_modules/.modules.yaml check-tauri-dev-port
$(PNPM) dev:tauri $(PNPM) dev:tauri
dev-fast: node_modules/.modules.yaml check-tauri-dev-port dev-fast: node_modules/.modules.yaml check-tauri-dev-port
$(PNPM) tauri dev -- --no-default-features $(PNPM) tauri dev -- --no-default-features --features duckdb-sidecar
dev-web: node_modules/.modules.yaml dev-web: node_modules/.modules.yaml
$(PNPM) dev:web $(PNPM) dev:web
+2 -2
View File
@@ -49,13 +49,13 @@ Each agent runs as a standalone process and communicates with DBX via stdin/stdo
## Multi-JRE Support ## Multi-JRE Support
Most Java agents target JRE 21. Native agents, such as `oracle`, `kingbase`, and `xugu`, do not require a JRE. DBX downloads and manages the JRE 21 installation automatically for Java agents. Most Java agents target JRE 21. Native agents, such as `duckdb`, `oracle`, `kingbase`, and `xugu`, do not require a JRE. DBX downloads and manages the JRE 21 installation automatically for Java agents.
## Choosing a Driver Language ## Choosing a Driver Language
For new agents, prefer a **native (Go or Rust) driver** over a Java/JDBC agent whenever a mature, license-compatible native driver is available. Native agents ship as a single self-contained executable with no JRE, which significantly reduces memory footprint and startup time — the JVM baseline that every Java agent pays even when idle is avoided entirely. For new agents, prefer a **native (Go or Rust) driver** over a Java/JDBC agent whenever a mature, license-compatible native driver is available. Native agents ship as a single self-contained executable with no JRE, which significantly reduces memory footprint and startup time — the JVM baseline that every Java agent pays even when idle is avoided entirely.
- **Native (Go/Rust)** — preferred when a usable native driver exists. See `drivers/oracle-go` (go-ora), `drivers/kingbase-go` (gokb), and `drivers/xugu` as reference implementations. No JRE download or management is needed. - **Native (C++/Go/Rust)** — preferred when a usable native driver exists. See `drivers/duckdb`, `drivers/oracle-go` (go-ora), `drivers/kingbase-go` (gokb), and `drivers/xugu` as reference implementations. No JRE download or management is needed.
- **Java/JDBC** — the default fallback when only a JDBC driver exists for the database, or when the native driver is immature or unmaintained. Most agents still fall in this category. - **Java/JDBC** — the default fallback when only a JDBC driver exists for the database, or when the native driver is immature or unmaintained. Most agents still fall in this category.
Native agents implement the same JSON-RPC contract and `versions.json` registration as Java agents; they ship an `agent` executable instead of `agent.jar`. If both native and Java source implementations exist for the same database, publish only the native artifact unless the Java variant has a separately registered compatibility profile, such as `oracle-legacy` / `oracle-10g`. Native agents implement the same JSON-RPC contract and `versions.json` registration as Java agents; they ship an `agent` executable instead of `agent.jar`. If both native and Java source implementations exist for the same database, publish only the native artifact unless the Java variant has a separately registered compatibility profile, such as `oracle-legacy` / `oracle-10g`.
+6 -2
View File
@@ -39,6 +39,8 @@ python3 scripts/validate_agent_jars.py
git diff --check git diff --check
``` ```
The package tests require the `zstd` CLI.
Expected result: Expected result:
- Python script tests pass. - Python script tests pass.
@@ -166,9 +168,11 @@ The release workflow will:
- Bump changed module versions in `versions.json`. - Bump changed module versions in `versions.json`.
- Build all agent shadow jars. - Build all agent shadow jars.
- Build/download JRE artifacts. - Build/download JRE `.tar.zst` artifacts.
- Generate `agent-registry.json`. - Generate `agent-registry.json`.
- Create a GitHub release with jars, JRE archives, and registry. - Create full offline platform ZIPs from raw staging files.
- Create one `.tar.zst` package per Java or native driver.
- Publish offline ZIPs, single-driver packages, JRE archives, and the registry.
## 9. Post-Release Verification ## 9. Post-Release Verification
File diff suppressed because it is too large. Load diff
+50
View File
@@ -0,0 +1,50 @@
[workspace]
[package]
name = "dbx-duckdb-driver"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
publish = false
[[bin]]
name = "duckdb-worker-hanging-connect-test-host"
path = "tests/support/duckdb_worker_hanging_connect_test_host.rs"
test = false
[[bin]]
name = "duckdb-worker-pid-test-host"
path = "tests/support/duckdb_worker_pid_test_host.rs"
test = false
[[bin]]
name = "duckdb-worker-file-lock-owner"
path = "tests/support/duckdb_worker_file_lock_owner.rs"
test = false
[dependencies]
chrono = "0.4"
duckdb = { version = "1.3.2", features = ["bundled", "parquet"] }
log = "0.4"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
tokio = { version = "1", features = ["full"] }
[dev-dependencies]
dbx-core = { path = "../../../crates/dbx-core", default-features = false, features = ["duckdb-sidecar"] }
sysinfo = "0.32"
tokio-util = "0.7"
uuid = { version = "1", features = ["v4"] }
[patch.crates-io]
tokio-postgres = { git = "https://github.com/t8y2/tokio-postgres-gaussdb.git", rev = "115f9fef10f0fc3669b5337955e4eb461fc349a6" }
postgres-types = { git = "https://github.com/t8y2/tokio-postgres-gaussdb.git", rev = "115f9fef10f0fc3669b5337955e4eb461fc349a6" }
postgres-protocol = { git = "https://github.com/t8y2/tokio-postgres-gaussdb.git", rev = "115f9fef10f0fc3669b5337955e4eb461fc349a6" }
mysql_async = { git = "https://github.com/t8y2/mysql_async.git", rev = "2be6e392eb9b06d20dcd2d8ed8eae748d413c9ec" }
[profile.release]
panic = "abort"
strip = true
lto = true
codegen-units = 1
opt-level = "s"
+42
View File
@@ -0,0 +1,42 @@
# DBX DuckDB Standalone Driver
This directory contains the standalone Rust DuckDB sidecar. It reuses DBX's
existing newline-delimited JSON worker runtime, while keeping DuckDB and
`libduckdb-sys` outside the main application dependency graph.
## Build
```bash
cargo build --manifest-path agents/drivers/duckdb/Cargo.toml --release --bin dbx-duckdb-driver
```
Point DBX at the resulting executable with:
```bash
DBX_DUCKDB_DRIVER_PATH=/absolute/path/to/dbx-duckdb-driver \
cargo run -p dbx --no-default-features --features duckdb-sidecar
```
Release builds publish this driver through the DBX driver registry. Driver
Manager installs it as `~/.dbx/agents/drivers/duckdb/agent` (or `agent.exe` on
Windows). `DBX_DUCKDB_DRIVER_PATH` remains available for local development.
## Release package
Each platform is published as a self-contained `.tar.zst` package used by both
online Driver Manager installation and manual single-driver import:
```text
agent-registry.json
drivers/dbx-agent-duckdb-<version>-<platform>[.exe]
```
DBX decompresses the package itself, so users do not need to install `zstd`,
DuckDB, or a separate database driver. The existing aggregate offline `.zip`
packages remain supported for backward compatibility.
## Current scope
The driver implements connect, execute, database/schema/table/column metadata,
table DDL, view source, completion assistance, attach, cancel, and shutdown over
the DBX sidecar protocol.
@@ -1,21 +1,17 @@
#![cfg(feature = "duckdb-bundled")]
use std::fs; use std::fs;
use std::path::Path; use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc; use std::sync::Arc;
use std::sync::Mutex; use std::sync::Mutex;
pub struct DuckDbConnection { pub struct DuckDbConnection {
connection: Mutex<duckdb::Connection>, connection: Mutex<duckdb::Connection>,
interrupt_handle: Arc<duckdb::InterruptHandle>, interrupt_handle: Arc<duckdb::InterruptHandle>,
draining: AtomicBool,
} }
impl DuckDbConnection { impl DuckDbConnection {
pub fn new(connection: duckdb::Connection) -> Self { pub fn new(connection: duckdb::Connection) -> Self {
let interrupt_handle = connection.interrupt_handle(); let interrupt_handle = connection.interrupt_handle();
Self { connection: Mutex::new(connection), interrupt_handle, draining: AtomicBool::new(false) } Self { connection: Mutex::new(connection), interrupt_handle }
} }
pub fn lock(&self) -> std::sync::LockResult<std::sync::MutexGuard<'_, duckdb::Connection>> { pub fn lock(&self) -> std::sync::LockResult<std::sync::MutexGuard<'_, duckdb::Connection>> {
@@ -26,19 +22,8 @@ impl DuckDbConnection {
self.interrupt_handle.clone() self.interrupt_handle.clone()
} }
pub fn mark_draining(&self) {
self.draining.store(true, Ordering::SeqCst);
}
pub fn clear_draining(&self) {
self.draining.store(false, Ordering::SeqCst);
}
pub fn is_draining(&self) -> bool {
self.draining.load(Ordering::SeqCst)
}
// Preserve PoisonError ownership so close_connection can still release the contained DuckDB handle. // Preserve PoisonError ownership so close_connection can still release the contained DuckDB handle.
#[cfg(test)]
#[allow(clippy::result_large_err)] #[allow(clippy::result_large_err)]
fn into_inner(self) -> std::sync::LockResult<duckdb::Connection> { fn into_inner(self) -> std::sync::LockResult<duckdb::Connection> {
self.connection.into_inner() self.connection.into_inner()
@@ -69,8 +54,8 @@ pub fn connect_path(path: &str) -> Result<Arc<DuckDbConnection>, String> {
/// Runs a connection init script statement-by-statement so a failure can /// Runs a connection init script statement-by-statement so a failure can
/// point at the offending statement instead of the whole batch. /// point at the offending statement instead of the whole batch.
pub fn run_init_script(con: &duckdb::Connection, script: &str) -> Result<(), String> { pub fn run_init_script(con: &duckdb::Connection, script: &str) -> Result<(), String> {
for (index, statement) in crate::db::duckdb_sql::split_sql_statements(script).iter().enumerate() { for (index, statement) in crate::sql::split_sql_statements(script).iter().enumerate() {
if crate::db::duckdb_sql::strip_leading_comments(statement).is_empty() { if crate::sql::strip_leading_comments(statement).is_empty() {
continue; continue;
} }
con.execute_batch(statement) con.execute_batch(statement)
@@ -131,7 +116,8 @@ pub fn is_memory_database_path(path: &str) -> bool {
/// Unlike relying on Drop, this calls `duckdb_disconnect` synchronously /// Unlike relying on Drop, this calls `duckdb_disconnect` synchronously
/// so the file handle is released before this function returns. /// so the file handle is released before this function returns.
/// On Windows this prevents "file already in use" errors when reconnecting. /// On Windows this prevents "file already in use" errors when reconnecting.
pub fn close_connection(con: Arc<DuckDbConnection>) { #[cfg(test)]
fn close_connection(con: Arc<DuckDbConnection>) {
match Arc::try_unwrap(con) { match Arc::try_unwrap(con) {
Ok(handle) => match handle.into_inner() { Ok(handle) => match handle.into_inner() {
Ok(conn) => { Ok(conn) => {
+8
View File
@@ -0,0 +1,8 @@
mod connection;
mod query;
mod runtime;
mod schema;
mod sql;
mod wire;
pub use runtime::run_stdio_worker;
+7
View File
@@ -0,0 +1,7 @@
#[tokio::main]
async fn main() {
if let Err(error) = dbx_duckdb_driver::run_stdio_worker().await {
eprintln!("DuckDB driver failed: {error}");
std::process::exit(1);
}
}
+436
View File
@@ -0,0 +1,436 @@
#![allow(clippy::items_after_test_module)]
use chrono::{DateTime, Duration as ChronoDuration, NaiveDate, NaiveDateTime, NaiveTime, Utc};
use duckdb::types::{TimeUnit, Value, ValueRef};
use crate::wire as db;
const MAX_ROWS: usize = 10000;
fn query_result_row_limit(max_rows: Option<usize>) -> usize {
max_rows.unwrap_or(MAX_ROWS).max(1)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn duckdb_execute_preserves_double_precision() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(
&con,
"SELECT 12.34567::DOUBLE AS sample, 0.5::DOUBLE AS half, 99.99::DOUBLE AS price, 1.0::DOUBLE AS one",
)
.expect("execute double query");
assert_eq!(result.columns, vec!["sample", "half", "price", "one"]);
let row = &result.rows[0];
assert_eq!(row[0], serde_json::json!(12.34567));
assert_eq!(row[1], serde_json::json!(0.5));
assert_eq!(row[2], serde_json::json!(99.99));
assert_eq!(row[3], serde_json::json!(1.0));
}
#[test]
fn duckdb_execute_create_insert_select_double() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
con.execute_batch("CREATE TABLE tmp1 (tmp_double DOUBLE)").expect("create table");
con.execute_batch("INSERT INTO tmp1 VALUES (45.678), (12.345), (99.999)").expect("insert");
let result = duckdb_execute(&con, "SELECT tmp_double FROM tmp1 ORDER BY tmp_double").expect("select doubles");
assert_eq!(result.rows.len(), 3);
assert_eq!(result.rows[0][0], serde_json::json!(12.345));
assert_eq!(result.rows[1][0], serde_json::json!(45.678));
assert_eq!(result.rows[2][0], serde_json::json!(99.999));
}
#[test]
fn duckdb_execute_returns_rows_for_from_first_query() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
con.execute_batch("CREATE TABLE users (id INTEGER, name VARCHAR)").expect("create table");
con.execute_batch("INSERT INTO users VALUES (2, 'Grace'), (1, 'Ada')").expect("insert");
let result = duckdb_execute(&con, "FROM users ORDER BY id").expect("execute from-first query");
assert_eq!(result.columns, vec!["id", "name"]);
assert_eq!(result.rows.len(), 2);
assert_eq!(result.rows[0], vec![serde_json::json!(1), serde_json::json!("Ada")]);
assert_eq!(result.rows[1], vec![serde_json::json!(2), serde_json::json!("Grace")]);
}
#[test]
fn duckdb_execute_returns_rows_for_summarize_query() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
con.execute_batch("CREATE TABLE metrics (value INTEGER)").expect("create table");
con.execute_batch("INSERT INTO metrics VALUES (1), (2), (NULL)").expect("insert");
let result = duckdb_execute(&con, "SUMMARIZE metrics").expect("execute summarize query");
assert!(!result.columns.is_empty());
assert!(!result.rows.is_empty());
}
#[test]
fn duckdb_execute_handles_various_types() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(
&con,
"SELECT 42 AS int_val, true AS bool_val, 'hello' AS text_val, 3.14::FLOAT AS float_val, 123456789012345::BIGINT AS big_val",
)
.expect("execute mixed types query");
let row = &result.rows[0];
assert_eq!(row[0], serde_json::json!(42));
assert_eq!(row[1], serde_json::json!(true));
assert_eq!(row[2], serde_json::Value::String("hello".to_string()));
assert!(row[3].is_number());
assert_eq!(row[4], serde_json::json!(123456789012345_i64));
}
#[test]
fn duckdb_execute_returns_list_values_as_json_arrays() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(&con, "SELECT ['a','b','c','d'];").expect("execute list query");
assert_eq!(result.rows, vec![vec![serde_json::json!(["a", "b", "c", "d"])]]);
}
#[test]
fn duckdb_execute_preserves_nulls_inside_list_values() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(&con, "SELECT [1, NULL, 3] AS items;").expect("execute nullable list query");
assert_eq!(result.columns, vec!["items"]);
assert_eq!(result.rows, vec![vec![serde_json::json!([1, null, 3])]]);
}
#[test]
fn duckdb_execute_returns_nested_complex_values_as_json() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(
&con,
"SELECT {'name': 'Ada', 'scores': [10, 20]} AS profile, MAP(['x', 'y'], [1, 2]) AS lookup, [1, 2, 3]::INTEGER[3] AS fixed_items",
)
.expect("execute complex values query");
assert_eq!(result.columns, vec!["profile", "lookup", "fixed_items"]);
assert_eq!(
result.rows,
vec![vec![
serde_json::json!({ "name": "Ada", "scores": [10, 20] }),
serde_json::json!([
{ "key": "x", "value": 1 },
{ "key": "y", "value": 2 },
]),
serde_json::json!([1, 2, 3]),
]]
);
}
#[test]
fn duckdb_execute_formats_temporal_values_by_column_type() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(
&con,
"SELECT DATE '2026-05-14' AS d, TIME '16:58:15' AS t, TIMESTAMP '2026-05-14 16:58:15.0' AS ts, NULL::TIMESTAMP AS nts",
)
.expect("execute temporal query");
assert_eq!(result.columns, vec!["d", "t", "ts", "nts"]);
assert_eq!(
result.rows,
vec![vec![
serde_json::Value::String("2026-05-14".to_string()),
serde_json::Value::String("16:58:15".to_string()),
serde_json::Value::String("2026-05-14 16:58:15".to_string()),
serde_json::Value::Null,
]]
);
}
}
#[cfg(test)]
pub fn duckdb_execute(con: &duckdb::Connection, sql: &str) -> Result<db::QueryResult, String> {
duckdb_execute_with_max_rows(con, sql, None)
}
fn duckdb_value_to_json(row: &duckdb::Row<'_>, idx: usize) -> serde_json::Value {
let Ok(value_ref) = row.get_ref(idx) else {
return serde_json::Value::Null;
};
match value_ref {
ValueRef::Null => serde_json::Value::Null,
ValueRef::Boolean(b) => serde_json::Value::Bool(b),
ValueRef::TinyInt(i) => serde_json::Value::Number((i as i64).into()),
ValueRef::SmallInt(i) => serde_json::Value::Number((i as i64).into()),
ValueRef::Int(i) => serde_json::Value::Number((i as i64).into()),
ValueRef::BigInt(i) => serde_json::Value::Number(i.into()),
ValueRef::HugeInt(i) => serde_json::Value::String(i.to_string()),
ValueRef::UTinyInt(i) => serde_json::Value::Number((i as u64).into()),
ValueRef::USmallInt(i) => serde_json::Value::Number((i as u64).into()),
ValueRef::UInt(i) => serde_json::Value::Number((i as u64).into()),
ValueRef::UBigInt(i) => serde_json::Value::Number(i.into()),
ValueRef::Float(f) => {
serde_json::Number::from_f64(f as f64).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null)
}
ValueRef::Double(f) => {
serde_json::Number::from_f64(f).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null)
}
ValueRef::Decimal(d) => serde_json::Value::String(d.to_string()),
ValueRef::Date32(days) => {
duckdb_date32_to_string(days).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
}
ValueRef::Time64(unit, value) => {
duckdb_time64_to_string(unit, value).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
}
ValueRef::Timestamp(unit, value) => {
duckdb_timestamp_to_string(unit, value).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
}
ValueRef::Text(bytes) => std::str::from_utf8(bytes)
.map(|s| serde_json::Value::String(s.to_string()))
.unwrap_or(serde_json::Value::Null),
ValueRef::Blob(bytes) => {
let hex: String = bytes.iter().map(|b| format!("{:02x}", b)).collect();
serde_json::Value::String(format!("\\x{hex}"))
}
ValueRef::Interval { months, days, nanos } => {
serde_json::Value::String(duckdb_interval_to_string(months, days, nanos))
}
ValueRef::List(..)
| ValueRef::Array(..)
| ValueRef::Struct(..)
| ValueRef::Map(..)
| ValueRef::Enum(..)
| ValueRef::Union(..) => duckdb_owned_value_to_json(&value_ref.to_owned()),
_ => duckdb_owned_value_to_json(&value_ref.to_owned()),
}
}
fn duckdb_owned_value_to_json(value: &Value) -> serde_json::Value {
match value {
Value::Null => serde_json::Value::Null,
Value::Boolean(b) => serde_json::Value::Bool(*b),
Value::TinyInt(i) => serde_json::Value::Number((*i as i64).into()),
Value::SmallInt(i) => serde_json::Value::Number((*i as i64).into()),
Value::Int(i) => serde_json::Value::Number((*i as i64).into()),
Value::BigInt(i) => serde_json::Value::Number((*i).into()),
Value::HugeInt(i) => serde_json::Value::String(i.to_string()),
Value::UTinyInt(i) => serde_json::Value::Number((*i as u64).into()),
Value::USmallInt(i) => serde_json::Value::Number((*i as u64).into()),
Value::UInt(i) => serde_json::Value::Number((*i as u64).into()),
Value::UBigInt(i) => serde_json::Value::Number((*i).into()),
Value::Float(f) => {
serde_json::Number::from_f64(*f as f64).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null)
}
Value::Double(f) => {
serde_json::Number::from_f64(*f).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null)
}
Value::Decimal(d) => serde_json::Value::String(d.to_string()),
Value::Timestamp(unit, value) => {
duckdb_timestamp_to_string(*unit, *value).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
}
Value::Text(text) | Value::Enum(text) => serde_json::Value::String(text.clone()),
Value::Blob(bytes) => {
let hex: String = bytes.iter().map(|b| format!("{:02x}", b)).collect();
serde_json::Value::String(format!("\\x{hex}"))
}
Value::Date32(days) => {
duckdb_date32_to_string(*days).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
}
Value::Time64(unit, value) => {
duckdb_time64_to_string(*unit, *value).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
}
Value::Interval { months, days, nanos } => {
serde_json::Value::String(duckdb_interval_to_string(*months, *days, *nanos))
}
Value::List(values) | Value::Array(values) => {
serde_json::Value::Array(values.iter().map(duckdb_owned_value_to_json).collect())
}
Value::Struct(entries) => serde_json::Value::Object(
entries.iter().map(|(key, value)| (key.clone(), duckdb_owned_value_to_json(value))).collect(),
),
Value::Map(entries) => serde_json::Value::Array(
entries
.iter()
.map(|(key, value)| {
serde_json::json!({
"key": duckdb_owned_value_to_json(key),
"value": duckdb_owned_value_to_json(value),
})
})
.collect(),
),
Value::Union(value) => duckdb_owned_value_to_json(value),
_ => serde_json::Value::Null,
}
}
fn duckdb_interval_to_string(months: i32, days: i32, nanos: i64) -> String {
let mut parts = Vec::new();
if months != 0 {
let years = months / 12;
let rem = months % 12;
if years != 0 {
parts.push(format!("{} year{}", years, if years.abs() != 1 { "s" } else { "" }));
}
if rem != 0 {
parts.push(format!("{} mon{}", rem, if rem.abs() != 1 { "s" } else { "" }));
}
}
if days != 0 {
parts.push(format!("{} day{}", days, if days.abs() != 1 { "s" } else { "" }));
}
if nanos != 0 {
let total_secs = nanos / 1_000_000_000;
let hours = total_secs / 3600;
let mins = (total_secs % 3600) / 60;
let secs = total_secs % 60;
let sub_nanos = (nanos % 1_000_000_000).unsigned_abs();
if sub_nanos > 0 {
parts.push(format!(
"{:02}:{:02}:{:02}.{}",
hours,
mins,
secs,
format_temporal_without_empty_fraction(format!("0.{:09}", sub_nanos)).trim_start_matches("0.")
));
} else {
parts.push(format!("{:02}:{:02}:{:02}", hours, mins, secs));
}
}
if parts.is_empty() {
"00:00:00".to_string()
} else {
parts.join(" ")
}
}
fn duckdb_date32_to_string(days: i32) -> Option<String> {
let epoch = NaiveDate::from_ymd_opt(1970, 1, 1)?;
epoch.checked_add_signed(ChronoDuration::days(i64::from(days))).map(|date| date.to_string())
}
fn duckdb_time64_to_string(unit: TimeUnit, value: i64) -> Option<String> {
let nanos = duckdb_time_unit_to_nanos(unit, value)?;
let seconds = nanos.div_euclid(1_000_000_000);
let nanos_remainder = nanos.rem_euclid(1_000_000_000) as u32;
if !(0..86_400).contains(&seconds) {
return None;
}
let time = NaiveTime::from_num_seconds_from_midnight_opt(seconds as u32, nanos_remainder)?;
Some(format_temporal_without_empty_fraction(time.to_string()))
}
fn duckdb_timestamp_to_string(unit: TimeUnit, value: i64) -> Option<String> {
let nanos = duckdb_time_unit_to_nanos(unit, value)?;
let seconds = nanos.div_euclid(1_000_000_000);
let nanos_remainder = nanos.rem_euclid(1_000_000_000) as u32;
let dt: DateTime<Utc> = DateTime::from_timestamp(seconds, nanos_remainder)?;
Some(format_naive_datetime(dt.naive_utc()))
}
fn duckdb_time_unit_to_nanos(unit: TimeUnit, value: i64) -> Option<i64> {
match unit {
TimeUnit::Second => value.checked_mul(1_000_000_000),
TimeUnit::Millisecond => value.checked_mul(1_000_000),
TimeUnit::Microsecond => value.checked_mul(1_000),
TimeUnit::Nanosecond => Some(value),
}
}
fn format_naive_datetime(value: NaiveDateTime) -> String {
if value.and_utc().timestamp_subsec_nanos() == 0 {
value.format("%Y-%m-%d %H:%M:%S").to_string()
} else {
format_temporal_without_empty_fraction(value.to_string())
}
}
fn format_temporal_without_empty_fraction(value: String) -> String {
if !value.contains('.') {
return value;
}
let trimmed = value.trim_end_matches('0').trim_end_matches('.');
trimmed.to_string()
}
pub fn duckdb_execute_with_max_rows(
con: &duckdb::Connection,
sql: &str,
max_rows: Option<usize>,
) -> Result<db::QueryResult, String> {
let start = std::time::Instant::now();
let row_limit = query_result_row_limit(max_rows);
if crate::sql::starts_with_duckdb_result_sql_keyword(sql) {
let mut stmt = con.prepare(sql).map_err(|e| e.to_string())?;
let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
let stmt_ref = rows.as_ref().ok_or("DuckDB statement unavailable")?;
let col_count = stmt_ref.column_count();
let columns: Vec<String> = (0..col_count)
.map(|i| stmt_ref.column_name(i).map(|s| s.to_string()).unwrap_or_else(|_| "?".to_string()))
.collect();
let mut result_rows = Vec::new();
while let Some(row) = rows.next().map_err(|e| e.to_string())? {
let vals: Vec<serde_json::Value> = (0..col_count).map(|i| duckdb_value_to_json(row, i)).collect();
result_rows.push(vals);
if result_rows.len() > row_limit {
break;
}
}
let truncated = result_rows.len() > row_limit;
if truncated {
result_rows.truncate(row_limit);
}
Ok(db::QueryResult {
columns,
column_types: Vec::new(),
column_sortables: vec![],
rows: result_rows,
affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(),
truncated,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
})
} else {
let affected = con.execute(sql, []).map_err(|e| e.to_string())?;
Ok(db::QueryResult {
columns: vec![],
column_types: Vec::new(),
column_sortables: vec![],
rows: vec![],
affected_rows: affected as u64,
execution_time_ms: start.elapsed().as_millis(),
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
})
}
}
pub(crate) fn duckdb_execute_for_database(
con: &duckdb::Connection,
attached_names: &[String],
database: Option<&str>,
sql: &str,
max_rows: Option<usize>,
) -> Result<db::QueryResult, String> {
if let Some(database) = database.map(str::trim).filter(|database| !database.is_empty()) {
let catalog = if database == "main" {
crate::schema::duckdb_primary_catalog(con, attached_names)?
} else {
database.to_string()
};
con.execute_batch(&format!("USE {}", duckdb_quote_ident(&catalog))).map_err(|e| e.to_string())?;
}
duckdb_execute_with_max_rows(con, sql, max_rows)
}
fn duckdb_quote_ident(value: &str) -> String {
format!("\"{}\"", value.replace('"', "\"\""))
}
@@ -1,28 +1,24 @@
#![cfg(feature = "duckdb-bundled")]
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Stdout}; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Stdout};
use crate::db; use crate::wire as db;
use crate::db::duckdb_worker_protocol::{ use crate::wire::{
DuckDbWorkerColumnParams, DuckDbWorkerConnectParams, DuckDbWorkerDatabaseParams, DuckDbWorkerError, AttachedDatabaseConfig, DuckDbWorkerColumnParams, DuckDbWorkerConnectParams, DuckDbWorkerDatabaseParams,
DuckDbWorkerExecuteParams, DuckDbWorkerMethod, DuckDbWorkerObjectSourceParams, DuckDbWorkerRequest, DuckDbWorkerError, DuckDbWorkerExecuteParams, DuckDbWorkerMethod, DuckDbWorkerObjectSourceParams,
DuckDbWorkerResponse, DuckDbWorkerTableParams, DuckDbWorkerRequest, DuckDbWorkerResponse, DuckDbWorkerTableParams,
}; };
use crate::models::connection::AttachedDatabaseConfig;
use crate::path_utils::expand_tilde;
#[derive(Default)] #[derive(Default)]
pub struct DuckDbWorkerSession { pub struct DuckDbWorkerSession {
connection: Option<Arc<db::duckdb_driver::DuckDbConnection>>, connection: Option<Arc<crate::connection::DuckDbConnection>>,
attached_names: Vec<String>, attached_names: Vec<String>,
} }
impl DuckDbWorkerSession { impl DuckDbWorkerSession {
pub fn connect(&mut self, params: DuckDbWorkerConnectParams) -> Result<(), String> { pub fn connect(&mut self, params: DuckDbWorkerConnectParams) -> Result<(), String> {
let path = expand_tilde(&params.path); let path = expand_tilde(&params.path);
let connection = db::duckdb_driver::connect_path(&path)?; let connection = crate::connection::connect_path(&path)?;
let mut attached_names = Vec::new(); let mut attached_names = Vec::new();
{ {
let locked = connection.lock().map_err(|e| e.to_string())?; let locked = connection.lock().map_err(|e| e.to_string())?;
@@ -36,7 +32,7 @@ impl DuckDbWorkerSession {
// instead of parsing the SQL, so ATTACH without an alias, // instead of parsing the SQL, so ATTACH without an alias,
// dynamic statements, and extension-specific syntax all count. // dynamic statements, and extension-specific syntax all count.
let before = duckdb_catalog_list(&locked)?; let before = duckdb_catalog_list(&locked)?;
db::duckdb_driver::run_init_script(&locked, script)?; crate::connection::run_init_script(&locked, script)?;
for name in duckdb_catalog_list(&locked)? { for name in duckdb_catalog_list(&locked)? {
if !before.iter().any(|existing| existing.eq_ignore_ascii_case(&name)) if !before.iter().any(|existing| existing.eq_ignore_ascii_case(&name))
&& !attached_names.iter().any(|attached| attached.eq_ignore_ascii_case(&name)) && !attached_names.iter().any(|attached| attached.eq_ignore_ascii_case(&name))
@@ -61,7 +57,7 @@ impl DuckDbWorkerSession {
&params.sql, &params.sql,
params.max_rows, params.max_rows,
)?; )?;
if let Some(name) = crate::db::duckdb_sql::attached_name_from_attach_sql(&params.sql) { if let Some(name) = crate::sql::attached_name_from_attach_sql(&params.sql) {
if !self.attached_names.iter().any(|attached| attached.eq_ignore_ascii_case(&name)) { if !self.attached_names.iter().any(|attached| attached.eq_ignore_ascii_case(&name)) {
self.attached_names.push(name); self.attached_names.push(name);
} }
@@ -104,6 +100,27 @@ impl DuckDbWorkerSession {
) )
} }
pub fn get_table_ddl(&self, params: DuckDbWorkerColumnParams) -> Result<String, String> {
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
let locked = connection.lock().map_err(|e| e.to_string())?;
crate::schema::duckdb_table_ddl_with_attached(
&locked,
&params.database,
&params.schema,
&params.table,
&self.attached_names,
)
}
pub fn completion_assistant(
&self,
request: db::CompletionAssistantRequest,
) -> Result<db::CompletionAssistantResponse, String> {
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
let locked = connection.lock().map_err(|e| e.to_string())?;
crate::schema::duckdb_completion_assistant_search(&locked, &request, &self.attached_names)
}
pub fn get_object_source(&self, params: DuckDbWorkerObjectSourceParams) -> Result<String, String> { pub fn get_object_source(&self, params: DuckDbWorkerObjectSourceParams) -> Result<String, String> {
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone(); let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
let locked = connection.lock().map_err(|e| e.to_string())?; let locked = connection.lock().map_err(|e| e.to_string())?;
@@ -153,6 +170,23 @@ impl DuckDbWorkerSession {
} }
} }
fn expand_tilde(path: &str) -> String {
if !path.starts_with('~') {
return path.to_string();
}
if path.len() == 1 {
return home_dir().unwrap_or_else(|| path.to_string());
}
if path.as_bytes().get(1) == Some(&b'/') {
return home_dir().map(|home| home + &path[1..]).unwrap_or_else(|| path.to_string());
}
path.to_string()
}
fn home_dir() -> Option<String> {
std::env::var("HOME").or_else(|_| std::env::var("USERPROFILE")).ok()
}
fn duckdb_catalog_list(con: &duckdb::Connection) -> Result<Vec<String>, String> { fn duckdb_catalog_list(con: &duckdb::Connection) -> Result<Vec<String>, String> {
let mut stmt = con.prepare("SHOW DATABASES").map_err(|e| e.to_string())?; let mut stmt = con.prepare("SHOW DATABASES").map_err(|e| e.to_string())?;
let rows = stmt.query_map([], |row| row.get::<_, String>(0)).map_err(|e| e.to_string())?; let rows = stmt.query_map([], |row| row.get::<_, String>(0)).map_err(|e| e.to_string())?;
@@ -196,6 +230,14 @@ impl DuckDbWorkerRuntime {
let params = request.parse_params()?; let params = request.parse_params()?;
Ok(DuckDbWorkerResponse::ok(request.id, session.list_columns(params)?)) Ok(DuckDbWorkerResponse::ok(request.id, session.list_columns(params)?))
}), }),
DuckDbWorkerMethod::GetTableDdl => self.handle_session_request(request, |session, request| {
let params = request.parse_params()?;
Ok(DuckDbWorkerResponse::ok(request.id, session.get_table_ddl(params)?))
}),
DuckDbWorkerMethod::CompletionAssistant => self.handle_session_request(request, |session, request| {
let params = request.parse_params()?;
Ok(DuckDbWorkerResponse::ok(request.id, session.completion_assistant(params)?))
}),
DuckDbWorkerMethod::GetObjectSource => self.handle_session_request(request, |session, request| { DuckDbWorkerMethod::GetObjectSource => self.handle_session_request(request, |session, request| {
let params = request.parse_params()?; let params = request.parse_params()?;
Ok(DuckDbWorkerResponse::ok(request.id, session.get_object_source(params)?)) Ok(DuckDbWorkerResponse::ok(request.id, session.get_object_source(params)?))
@@ -422,7 +464,10 @@ mod tests {
.expect("connect"); .expect("connect");
session session
.execute(DuckDbWorkerExecuteParams { .execute(DuckDbWorkerExecuteParams {
sql: "CREATE TABLE users (id INTEGER PRIMARY KEY, name VARCHAR)".to_string(), sql: "CREATE TABLE users (id INTEGER PRIMARY KEY, name VARCHAR); \
CREATE SCHEMA analytics; \
CREATE TABLE analytics.items (id INTEGER)"
.to_string(),
database: None, database: None,
max_rows: None, max_rows: None,
}) })
@@ -440,10 +485,18 @@ mod tests {
table: "users".to_string(), table: "users".to_string(),
}) })
.expect("list columns"); .expect("list columns");
let analytics_columns = session
.list_columns(DuckDbWorkerColumnParams {
database: "main".to_string(),
schema: "analytics".to_string(),
table: "items".to_string(),
})
.expect("list analytics columns");
assert!(schemas.iter().any(|schema| schema == "main")); assert!(schemas.iter().any(|schema| schema == "main"));
assert!(tables.iter().any(|table| table.name == "users")); assert!(tables.iter().any(|table| table.name == "users"));
assert!(columns.iter().any(|column| column.name == "id" && column.is_primary_key)); assert!(columns.iter().any(|column| column.name == "id" && column.is_primary_key));
assert_eq!(analytics_columns.iter().map(|column| column.name.as_str()).collect::<Vec<_>>(), vec!["id"]);
} }
#[test] #[test]
+792
View File
@@ -0,0 +1,792 @@
use std::collections::HashMap;
use crate::wire as db;
#[cfg(test)]
pub fn duckdb_query_tables_in_database(
con: &duckdb::Connection,
database: &str,
schema: &str,
) -> Result<Vec<db::TableInfo>, String> {
duckdb_query_tables_in_database_with_attached(con, database, schema, &[])
}
pub fn duckdb_query_tables_in_database_with_attached(
con: &duckdb::Connection,
database: &str,
schema: &str,
attached_names: &[String],
) -> Result<Vec<db::TableInfo>, String> {
let database = duckdb_catalog_name(con, database, attached_names)?;
let mut stmt = con.prepare(
"SELECT table_name, table_type FROM information_schema.tables WHERE table_catalog = ? AND table_schema = ? ORDER BY table_name"
).map_err(|e| e.to_string())?;
let rows = stmt
.query_map((database.as_str(), schema), |row| {
Ok(db::TableInfo {
name: row.get::<_, String>(0)?,
table_type: row.get::<_, String>(1)?,
comment: None,
parent_schema: None,
parent_name: None,
})
})
.map_err(|e| e.to_string())?;
let tables: Vec<db::TableInfo> = rows.filter_map(|r| r.ok()).collect();
if tables.is_empty() && duckdb_is_quack_catalog(con, &database) {
if let Some(remote) = duckdb_quack_remote_tables(con, &database, schema) {
return Ok(remote);
}
}
Ok(tables)
}
pub fn duckdb_attach_database(con: &duckdb::Connection, name: &str, path: &str) -> Result<(), String> {
let name = name.trim();
let path = path.trim();
if name.is_empty() || path.is_empty() {
return Err("DuckDB attached database name and path are required".to_string());
}
let sql = format!("ATTACH {} AS {}", duckdb_quote_string(path), duckdb_quote_ident(name));
con.execute_batch(&sql).map_err(|e| format!("Failed to attach database \"{name}\": {e}"))
}
#[cfg(test)]
pub fn duckdb_list_databases(con: &duckdb::Connection) -> Result<Vec<db::DatabaseInfo>, String> {
duckdb_list_databases_with_attached(con, &[])
}
pub fn duckdb_list_databases_with_attached(
con: &duckdb::Connection,
attached_names: &[String],
) -> Result<Vec<db::DatabaseInfo>, String> {
let primary = duckdb_primary_catalog(con, attached_names)?;
let mut stmt = con.prepare("SHOW DATABASES").map_err(|e| e.to_string())?;
let rows = stmt
.query_map([], |row| {
let name = row.get::<_, String>(0)?;
Ok(db::DatabaseInfo { name: if name == primary { "main".to_string() } else { name } })
})
.map_err(|e| e.to_string())?;
Ok(rows.filter_map(|row| row.ok()).collect())
}
pub fn duckdb_list_schemas_with_attached(
con: &duckdb::Connection,
database: &str,
attached_names: &[String],
) -> Result<Vec<String>, String> {
let database = duckdb_catalog_name(con, database, attached_names)?;
let mut stmt = con
.prepare(
"SELECT schema_name FROM information_schema.schemata WHERE catalog_name = ? AND schema_name NOT IN ('information_schema', 'pg_catalog') ORDER BY schema_name",
)
.map_err(|e| e.to_string())?;
let rows = stmt.query_map([database.as_str()], |row| row.get::<_, String>(0)).map_err(|e| e.to_string())?;
let schemas: Vec<String> = rows.filter_map(|r| r.ok()).collect();
if schemas.is_empty() && duckdb_is_quack_catalog(con, &database) {
if let Some(remote) = duckdb_quack_remote_schemas(con, &database) {
return Ok(remote);
}
}
Ok(schemas)
}
pub fn duckdb_object_source_with_attached(
con: &duckdb::Connection,
database: &str,
schema: &str,
name: &str,
object_type: &db::ObjectSourceKind,
attached_names: &[String],
) -> Result<String, String> {
if !matches!(object_type, db::ObjectSourceKind::View) {
return Err("DuckDB object source only supports views".to_string());
}
let database = duckdb_catalog_name(con, database, attached_names)?;
let mut stmt = con
.prepare("SELECT sql FROM duckdb_views() WHERE database_name = ? AND schema_name = ? AND view_name = ?")
.map_err(|e| e.to_string())?;
let mut rows =
stmt.query_map((database.as_str(), schema, name), |row| row.get::<_, String>(0)).map_err(|e| e.to_string())?;
match rows.next() {
Some(row) => row.map_err(|e| e.to_string()),
None => Err(format!("DuckDB view source not found: {database}.{schema}.{name}")),
}
}
pub fn duckdb_table_ddl_with_attached(
con: &duckdb::Connection,
database: &str,
schema: &str,
table: &str,
attached_names: &[String],
) -> Result<String, String> {
let database = duckdb_catalog_name(con, database, attached_names)?;
let mut stmt = con
.prepare("SELECT sql FROM duckdb_tables() WHERE database_name = ? AND schema_name = ? AND table_name = ?")
.map_err(|e| e.to_string())?;
let mut rows =
stmt.query_map((database.as_str(), schema, table), |row| row.get::<_, String>(0)).map_err(|e| e.to_string())?;
match rows.next() {
Some(row) => row.map_err(|e| e.to_string()),
None => Err(format!("DuckDB table not found: {database}.{schema}.{table}")),
}
}
/// Identifies quack catalogs by the storage-extension `type` that
/// `duckdb_databases()` reports, rather than by the attach aliases parsed from
/// the init script: `ATTACH 'quack:host:port'` without an `AS` alias gets a
/// derived catalog name (e.g. `localhost:9494`) that no parser sees.
fn duckdb_is_quack_catalog(con: &duckdb::Connection, database: &str) -> bool {
con.query_row("SELECT type FROM duckdb_databases() WHERE lower(database_name) = lower(?)", [database], |row| {
row.get::<_, String>(0)
})
.map(|catalog_type| catalog_type.eq_ignore_ascii_case("quack"))
.unwrap_or(false)
}
/// Quack-attached catalogs expose no metadata on the client side (the beta
/// extension implements name resolution and streaming scans, but not catalog
/// enumeration: `information_schema`, `duckdb_tables()` and `SHOW ALL TABLES`
/// all skip the remote catalog). The extension's `quack_query_by_name` table
/// function runs SQL on the remote server, so when a metadata query for an
/// attached catalog comes back empty we ask the remote for its own
/// information_schema. For non-quack catalogs (or when the extension is not
/// loaded) the function call errors and the fallback quietly yields `None`.
fn duckdb_quack_remote_query<T>(
con: &duckdb::Connection,
alias: &str,
remote_sql: &str,
map_row: impl Fn(&duckdb::Row<'_>) -> Result<T, duckdb::Error>,
) -> Option<Vec<T>> {
let sql = format!(
"SELECT * FROM quack_query_by_name({}, {})",
duckdb_quote_string(alias),
duckdb_quote_string(remote_sql)
);
let mut stmt = match con.prepare(&sql) {
Ok(stmt) => stmt,
Err(e) => {
log::debug!("quack metadata fallback unavailable for catalog {alias}: {e}");
return None;
}
};
let rows = match stmt.query_map([], |row| map_row(row)) {
Ok(rows) => rows,
Err(e) => {
log::debug!("quack metadata fallback query failed for catalog {alias}: {e}");
return None;
}
};
Some(rows.filter_map(|r| r.ok()).collect())
}
fn duckdb_quack_remote_schemas(con: &duckdb::Connection, alias: &str) -> Option<Vec<String>> {
duckdb_quack_remote_query(
con,
alias,
"SELECT DISTINCT schema_name FROM information_schema.schemata WHERE schema_name NOT IN ('information_schema', 'pg_catalog') ORDER BY schema_name",
|row| row.get::<_, String>(0),
)
}
fn duckdb_quack_remote_tables(con: &duckdb::Connection, alias: &str, schema: &str) -> Option<Vec<db::TableInfo>> {
let remote_sql = format!(
"SELECT table_name, table_type FROM information_schema.tables WHERE table_schema = {} ORDER BY table_name",
duckdb_quote_string(schema)
);
duckdb_quack_remote_query(con, alias, &remote_sql, |row| {
Ok(db::TableInfo {
name: row.get::<_, String>(0)?,
table_type: row.get::<_, String>(1)?,
comment: None,
parent_schema: None,
parent_name: None,
})
})
}
fn duckdb_quack_remote_columns(
con: &duckdb::Connection,
alias: &str,
schema: &str,
table: &str,
) -> Option<Vec<db::ColumnInfo>> {
let pk_sql = format!(
"SELECT kcu.column_name
FROM information_schema.table_constraints tc
JOIN information_schema.key_column_usage kcu
ON tc.constraint_name = kcu.constraint_name
AND tc.table_schema = kcu.table_schema
AND tc.table_name = kcu.table_name
WHERE tc.constraint_type = 'PRIMARY KEY'
AND tc.table_schema = {schema}
AND tc.table_name = {table}
ORDER BY kcu.ordinal_position",
schema = duckdb_quote_string(schema),
table = duckdb_quote_string(table),
);
let primary_keys: std::collections::HashSet<String> =
duckdb_quack_remote_query(con, alias, &pk_sql, |row| row.get::<_, String>(0))
.map(|names| names.into_iter().collect())
.unwrap_or_default();
let columns_sql = format!(
"SELECT column_name, data_type, is_nullable, column_default
FROM information_schema.columns
WHERE table_schema = {schema} AND table_name = {table}
ORDER BY ordinal_position",
schema = duckdb_quote_string(schema),
table = duckdb_quote_string(table),
);
duckdb_quack_remote_query(con, alias, &columns_sql, |row| {
let name = row.get::<_, String>(0)?;
Ok(db::ColumnInfo {
is_primary_key: primary_keys.contains(&name),
name,
data_type: row.get::<_, String>(1)?,
is_nullable: row.get::<_, String>(2).unwrap_or_default() == "YES",
column_default: row.get::<_, Option<String>>(3)?,
extra: None,
comment: None,
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
..Default::default()
})
})
}
fn duckdb_catalog_name(con: &duckdb::Connection, database: &str, attached_names: &[String]) -> Result<String, String> {
if database.trim().is_empty() || database == "main" {
return duckdb_primary_catalog(con, attached_names);
}
Ok(database.to_string())
}
pub fn duckdb_primary_catalog(con: &duckdb::Connection, attached_names: &[String]) -> Result<String, String> {
if attached_names.is_empty() {
return duckdb_current_database(con);
}
let attached: std::collections::HashSet<String> = attached_names.iter().map(|name| name.to_lowercase()).collect();
let mut stmt = con.prepare("SHOW DATABASES").map_err(|e| e.to_string())?;
let rows = stmt.query_map([], |row| row.get::<_, String>(0)).map_err(|e| e.to_string())?;
for row in rows {
let name = row.map_err(|e| e.to_string())?;
if !attached.contains(&name.to_lowercase()) {
return Ok(name);
}
}
duckdb_current_database(con)
}
fn duckdb_current_database(con: &duckdb::Connection) -> Result<String, String> {
con.query_row("SELECT current_database()", [], |row| row.get::<_, String>(0)).map_err(|e| e.to_string())
}
fn duckdb_quote_ident(value: &str) -> String {
format!("\"{}\"", value.replace('"', "\"\""))
}
fn duckdb_quote_string(value: &str) -> String {
format!("'{}'", value.replace('\'', "''"))
}
#[cfg(test)]
pub fn duckdb_query_columns(con: &duckdb::Connection, table: &str) -> Result<Vec<db::ColumnInfo>, String> {
duckdb_query_columns_in_database(con, "main", "main", table)
}
#[cfg(test)]
pub fn duckdb_query_columns_in_database(
con: &duckdb::Connection,
database: &str,
schema: &str,
table: &str,
) -> Result<Vec<db::ColumnInfo>, String> {
duckdb_query_columns_in_database_with_attached(con, database, schema, table, &[])
}
pub fn duckdb_query_columns_in_database_with_attached(
con: &duckdb::Connection,
database: &str,
schema: &str,
table: &str,
attached_names: &[String],
) -> Result<Vec<db::ColumnInfo>, String> {
let database = duckdb_catalog_name(con, database, attached_names)?;
let mut pk_stmt = con
.prepare(
"SELECT kcu.column_name
FROM information_schema.table_constraints tc
JOIN information_schema.key_column_usage kcu
ON tc.constraint_name = kcu.constraint_name
AND tc.table_schema = kcu.table_schema
AND tc.table_name = kcu.table_name
WHERE tc.constraint_type = 'PRIMARY KEY'
AND tc.table_catalog = ?
AND tc.table_schema = ?
AND tc.table_name = ?
ORDER BY kcu.ordinal_position",
)
.map_err(|e| e.to_string())?;
let pk_rows = pk_stmt
.query_map((database.as_str(), schema, table), |row| row.get::<_, String>(0))
.map_err(|e| e.to_string())?;
let primary_keys: std::collections::HashSet<String> = pk_rows.filter_map(|r| r.ok()).collect();
let column_comments = duckdb_column_comments(con, &database, schema, table);
let mut stmt = con
.prepare(
"SELECT column_name, data_type, is_nullable, column_default
FROM information_schema.columns
WHERE table_catalog = ? AND table_schema = ? AND table_name = ?
ORDER BY ordinal_position",
)
.map_err(|e| e.to_string())?;
let rows = stmt
.query_map((database.as_str(), schema, table), |row| {
let name = row.get::<_, String>(0)?;
let comment = column_comments.get(&name).cloned().flatten();
Ok(db::ColumnInfo {
is_primary_key: primary_keys.contains(&name),
name,
data_type: row.get::<_, String>(1)?,
is_nullable: row.get::<_, String>(2).unwrap_or_default() == "YES",
column_default: row.get::<_, Option<String>>(3)?,
extra: None,
comment,
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
..Default::default()
})
})
.map_err(|e| e.to_string())?;
let columns: Vec<db::ColumnInfo> = rows.filter_map(|r| r.ok()).collect();
if columns.is_empty() && duckdb_is_quack_catalog(con, &database) {
if let Some(remote) = duckdb_quack_remote_columns(con, &database, schema, table) {
return Ok(remote);
}
}
Ok(columns)
}
fn duckdb_column_comments(
con: &duckdb::Connection,
database: &str,
schema: &str,
table: &str,
) -> HashMap<String, Option<String>> {
let Ok(mut stmt) = con.prepare(
"SELECT column_name, comment FROM duckdb_columns() \
WHERE database_name = ? AND schema_name = ? AND table_name = ?",
) else {
// Older DuckDB versions may not expose the comment column; keep metadata browsing functional.
return HashMap::new();
};
let Ok(rows) = stmt
.query_map((database, schema, table), |row| Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?)))
else {
return HashMap::new();
};
rows.filter_map(Result::ok).collect()
}
pub fn duckdb_completion_assistant_search(
con: &duckdb::Connection,
request: &db::CompletionAssistantRequest,
attached_names: &[String],
) -> Result<db::CompletionAssistantResponse, String> {
let limit = request.max_results.unwrap_or(100).clamp(1, 1000);
let kinds = if request.object_kinds.is_empty() {
vec![db::CompletionAssistantObjectKind::Table, db::CompletionAssistantObjectKind::View]
} else {
request.object_kinds.clone()
};
let mut candidates = Vec::new();
if kinds.iter().any(|kind| matches!(kind, db::CompletionAssistantObjectKind::Schema)) {
candidates.extend(duckdb_completion_schemas(con, request, attached_names, limit)?);
if candidates.len() >= limit {
return Ok(db::CompletionAssistantResponse { candidates, incomplete: true, fallback_used: false });
}
}
if kinds.iter().any(db::CompletionAssistantObjectKind::is_table_like) {
candidates.extend(duckdb_completion_tables(con, request, &kinds, attached_names, limit - candidates.len())?);
if candidates.len() >= limit {
return Ok(db::CompletionAssistantResponse { candidates, incomplete: true, fallback_used: false });
}
}
if kinds.iter().any(|kind| matches!(kind, db::CompletionAssistantObjectKind::Column)) {
candidates.extend(duckdb_completion_columns(con, request, attached_names, limit - candidates.len())?);
if candidates.len() >= limit {
return Ok(db::CompletionAssistantResponse { candidates, incomplete: true, fallback_used: false });
}
}
Ok(db::CompletionAssistantResponse { candidates, incomplete: false, fallback_used: false })
}
fn duckdb_completion_schemas(
con: &duckdb::Connection,
request: &db::CompletionAssistantRequest,
attached_names: &[String],
limit: usize,
) -> Result<Vec<db::CompletionAssistantCandidate>, String> {
if limit == 0 {
return Ok(Vec::new());
}
let database = duckdb_catalog_name(con, &request.database, attached_names)?;
let pattern = duckdb_completion_like_pattern(request);
let mut stmt = con
.prepare(
"SELECT schema_name
FROM information_schema.schemata
WHERE catalog_name = ?
AND schema_name NOT IN ('information_schema', 'pg_catalog')
AND lower(schema_name) LIKE lower(?) ESCAPE '\\'
ORDER BY schema_name
LIMIT ?",
)
.map_err(|e| e.to_string())?;
let rows = stmt
.query_map((database.as_str(), pattern.as_str(), limit as i64), |row| {
let schema = row.get::<_, String>(0)?;
Ok(db::CompletionAssistantCandidate {
name: schema.clone(),
kind: db::CompletionAssistantCandidateKind::Schema,
database: Some(request.database.clone()),
schema: Some(schema),
parent_schema: None,
parent_name: None,
comment: None,
data_type: None,
signature: None,
})
})
.map_err(|e| e.to_string())?;
Ok(rows.filter_map(|row| row.ok()).collect())
}
fn duckdb_completion_tables(
con: &duckdb::Connection,
request: &db::CompletionAssistantRequest,
kinds: &[db::CompletionAssistantObjectKind],
attached_names: &[String],
limit: usize,
) -> Result<Vec<db::CompletionAssistantCandidate>, String> {
if limit == 0 {
return Ok(Vec::new());
}
let database = duckdb_catalog_name(con, &request.database, attached_names)?;
let schema = request.parent_schema.as_deref().or(request.schema.as_deref()).unwrap_or("main");
let include_tables = kinds.iter().any(|kind| matches!(kind, db::CompletionAssistantObjectKind::Table));
let include_views = kinds.iter().any(|kind| matches!(kind, db::CompletionAssistantObjectKind::View));
let pattern = duckdb_completion_like_pattern(request);
let mut stmt = con
.prepare(
"SELECT table_name, table_type
FROM information_schema.tables
WHERE table_catalog = ?
AND table_schema = ?
AND ((? AND table_type = 'BASE TABLE') OR (? AND table_type = 'VIEW'))
AND lower(table_name) LIKE lower(?) ESCAPE '\\'
ORDER BY table_name
LIMIT ?",
)
.map_err(|e| e.to_string())?;
let rows = stmt
.query_map((database.as_str(), schema, include_tables, include_views, pattern.as_str(), limit as i64), |row| {
let table_type = row.get::<_, String>(1)?;
Ok(db::CompletionAssistantCandidate {
name: row.get(0)?,
kind: if table_type.eq_ignore_ascii_case("VIEW") {
db::CompletionAssistantCandidateKind::View
} else {
db::CompletionAssistantCandidateKind::Table
},
database: Some(request.database.clone()),
schema: Some(schema.to_string()),
parent_schema: None,
parent_name: None,
comment: None,
data_type: None,
signature: None,
})
})
.map_err(|e| e.to_string())?;
Ok(rows.filter_map(|row| row.ok()).collect())
}
fn duckdb_completion_columns(
con: &duckdb::Connection,
request: &db::CompletionAssistantRequest,
attached_names: &[String],
limit: usize,
) -> Result<Vec<db::CompletionAssistantCandidate>, String> {
if limit == 0 {
return Ok(Vec::new());
}
let Some(table) = request.parent_name.as_deref().filter(|table| !table.trim().is_empty()) else {
return Ok(Vec::new());
};
let database = duckdb_catalog_name(con, &request.database, attached_names)?;
let schema = request.parent_schema.as_deref().or(request.schema.as_deref()).unwrap_or("main");
let pattern = duckdb_completion_like_pattern(request);
let mut stmt = con
.prepare(
"SELECT column_name, data_type
FROM information_schema.columns
WHERE table_catalog = ?
AND table_schema = ?
AND table_name = ?
AND lower(column_name) LIKE lower(?) ESCAPE '\\'
ORDER BY ordinal_position
LIMIT ?",
)
.map_err(|e| e.to_string())?;
let rows = stmt
.query_map((database.as_str(), schema, table, pattern.as_str(), limit as i64), |row| {
Ok(db::CompletionAssistantCandidate {
name: row.get(0)?,
kind: db::CompletionAssistantCandidateKind::Column,
database: Some(request.database.clone()),
schema: Some(schema.to_string()),
parent_schema: Some(schema.to_string()),
parent_name: Some(table.to_string()),
comment: None,
data_type: Some(row.get(1)?),
signature: None,
})
})
.map_err(|e| e.to_string())?;
Ok(rows.filter_map(|row| row.ok()).collect())
}
fn duckdb_completion_like_pattern(request: &db::CompletionAssistantRequest) -> String {
let mask = request.mask.trim().trim_matches('%');
let escaped = mask.replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_");
match request.match_mode.as_ref().unwrap_or(&db::CompletionAssistantMatchMode::Prefix) {
db::CompletionAssistantMatchMode::Prefix => format!("{escaped}%"),
db::CompletionAssistantMatchMode::Contains => format!("%{escaped}%"),
}
}
#[cfg(test)]
mod tests {
use super::*;
use dbx_core::models::connection::DatabaseType;
#[test]
fn duckdb_list_databases_includes_attached_database() {
let unique = uuid::Uuid::new_v4();
let path = std::env::temp_dir().join(format!("dbx-attached-{unique}.duckdb"));
let _ = std::fs::remove_file(&path);
let con = duckdb::Connection::open_in_memory().unwrap();
duckdb_attach_database(&con, "analytics", path.to_str().unwrap()).unwrap();
let databases = duckdb_list_databases(&con).unwrap();
assert!(databases.iter().any(|database| database.name == "main"));
assert!(databases.iter().any(|database| database.name == "analytics"));
let _ = std::fs::remove_file(path);
}
#[test]
fn duckdb_query_tables_filters_by_attached_database() {
let unique = uuid::Uuid::new_v4();
let path = std::env::temp_dir().join(format!("dbx-attached-tables-{unique}.duckdb"));
let _ = std::fs::remove_file(&path);
let con = duckdb::Connection::open_in_memory().unwrap();
con.execute_batch("CREATE TABLE main_table(id INTEGER);").unwrap();
duckdb_attach_database(&con, "analytics", path.to_str().unwrap()).unwrap();
con.execute_batch("CREATE TABLE analytics.attached_table(id INTEGER);").unwrap();
let main_tables = duckdb_query_tables_in_database(&con, "main", "main").unwrap();
let attached_tables = duckdb_query_tables_in_database(&con, "analytics", "main").unwrap();
assert!(main_tables.iter().any(|table| table.name == "main_table"));
assert!(!main_tables.iter().any(|table| table.name == "attached_table"));
assert!(attached_tables.iter().any(|table| table.name == "attached_table"));
assert!(!attached_tables.iter().any(|table| table.name == "main_table"));
let _ = std::fs::remove_file(path);
}
#[test]
fn duckdb_query_columns_includes_column_comments() {
let con = duckdb::Connection::open_in_memory().unwrap();
con.execute_batch(
"CREATE TABLE users (id INTEGER, name VARCHAR); \
COMMENT ON COLUMN users.name IS 'Display name';",
)
.unwrap();
let columns = duckdb_query_columns(&con, "users").unwrap();
let name = columns.iter().find(|column| column.name == "name").unwrap();
assert_eq!(name.comment.as_deref(), Some("Display name"));
}
#[test]
fn duckdb_table_ddl_filters_by_database_schema_and_table_name() {
let unique = uuid::Uuid::new_v4();
let path = std::env::temp_dir().join(format!("dbx-attached-ddl-{unique}.duckdb"));
let _ = std::fs::remove_file(&path);
let con = duckdb::Connection::open_in_memory().unwrap();
con.execute_batch(
"CREATE SCHEMA reporting; \
CREATE TABLE main.orders(main_id INTEGER); \
CREATE TABLE reporting.orders(reporting_id INTEGER);",
)
.unwrap();
duckdb_attach_database(&con, "analytics", path.to_str().unwrap()).unwrap();
con.execute_batch("CREATE TABLE analytics.orders(attached_id INTEGER);").unwrap();
let main_ddl =
duckdb_table_ddl_with_attached(&con, "main", "main", "orders", &["analytics".to_string()]).unwrap();
let reporting_ddl =
duckdb_table_ddl_with_attached(&con, "main", "reporting", "orders", &["analytics".to_string()]).unwrap();
let attached_ddl =
duckdb_table_ddl_with_attached(&con, "analytics", "main", "orders", &["analytics".to_string()]).unwrap();
assert!(main_ddl.contains("main_id"));
assert!(reporting_ddl.contains("reporting_id"));
assert!(attached_ddl.contains("attached_id"));
let _ = std::fs::remove_file(path);
}
#[test]
fn duckdb_completion_assistant_searches_catalog_metadata_with_limit() {
let con = duckdb::Connection::open_in_memory().unwrap();
con.execute_batch(
"CREATE TABLE account(id INTEGER, display_name VARCHAR); \
CREATE VIEW account_view AS SELECT id FROM account;",
)
.unwrap();
let request = db::CompletionAssistantRequest {
connection_id: "c1".to_string(),
database: "main".to_string(),
schema: Some("main".to_string()),
object_kinds: vec![db::CompletionAssistantObjectKind::Table, db::CompletionAssistantObjectKind::View],
mask: "account".to_string(),
case_sensitive: false,
global_search: false,
max_results: Some(1),
search_in_comments: false,
search_in_definitions: false,
parent_schema: Some("main".to_string()),
parent_name: None,
match_mode: Some(db::CompletionAssistantMatchMode::Prefix),
};
let tables = duckdb_completion_assistant_search(&con, &request, &[]).unwrap();
assert_eq!(tables.candidates.len(), 1);
assert!(tables.incomplete);
assert!(!tables.fallback_used);
assert_eq!(tables.candidates[0].name, "account");
let columns = duckdb_completion_assistant_search(
&con,
&db::CompletionAssistantRequest {
object_kinds: vec![db::CompletionAssistantObjectKind::Column],
mask: "name".to_string(),
max_results: Some(10),
parent_name: Some("account".to_string()),
match_mode: Some(db::CompletionAssistantMatchMode::Contains),
..request
},
&[],
)
.unwrap();
assert_eq!(columns.candidates.len(), 1);
assert_eq!(columns.candidates[0].name, "display_name");
}
#[test]
fn duckdb_object_source_filters_by_schema_and_view_name() {
let con = duckdb::Connection::open_in_memory().unwrap();
con.execute_batch(
"CREATE SCHEMA reporting; \
CREATE VIEW main.active_orders AS SELECT 'main' AS origin; \
CREATE VIEW reporting.active_orders AS SELECT 'reporting' AS origin;",
)
.unwrap();
let main_source =
duckdb_object_source_with_attached(&con, "main", "main", "active_orders", &db::ObjectSourceKind::View, &[])
.unwrap();
let reporting_source = duckdb_object_source_with_attached(
&con,
"main",
"reporting",
"active_orders",
&db::ObjectSourceKind::View,
&[],
)
.unwrap();
assert!(main_source.contains("'main'"));
assert!(reporting_source.contains("'reporting'"));
}
#[test]
fn duckdb_object_source_rejects_unsupported_or_missing_objects() {
let con = duckdb::Connection::open_in_memory().unwrap();
let unsupported = duckdb_object_source_with_attached(
&con,
"main",
"main",
"calculate_total",
&db::ObjectSourceKind::Function,
&[],
)
.unwrap_err();
let missing =
duckdb_object_source_with_attached(&con, "main", "main", "missing_view", &db::ObjectSourceKind::View, &[])
.unwrap_err();
assert_eq!(unsupported, "DuckDB object source only supports views");
assert!(missing.contains("DuckDB view source not found"));
}
#[test]
fn duckdb_catalog_view_source_can_replace_existing_view() {
let con = duckdb::Connection::open_in_memory().unwrap();
con.execute_batch("CREATE VIEW active_orders AS SELECT 1 AS value;").unwrap();
let source =
duckdb_object_source_with_attached(&con, "main", "main", "active_orders", &db::ObjectSourceKind::View, &[])
.unwrap();
let edited_source = source.replace("SELECT 1", "SELECT 2");
assert_ne!(edited_source, source);
let statements = dbx_core::object_source_sql::build_executable_object_source_statements(
dbx_core::object_source_sql::EditableObjectSourceSqlInput {
database_type: DatabaseType::DuckDb,
object_type: dbx_core::db::ObjectSourceKind::View,
schema: Some("main".to_string()),
name: "active_orders".to_string(),
source: edited_source,
},
)
.unwrap();
con.execute_batch(&statements.join("\n")).unwrap();
let value = con.query_row("SELECT value FROM active_orders", [], |row| row.get::<_, i32>(0)).unwrap();
assert_eq!(value, 2);
}
}
+274
View File
@@ -0,0 +1,274 @@
pub fn split_sql_statements(script: &str) -> Vec<String> {
let bytes = script.as_bytes();
let mut statements = Vec::new();
let mut start = 0;
let mut index = 0;
let mut in_single = false;
let mut in_double = false;
let mut in_escape_single = false;
while index < bytes.len() {
match bytes[index] {
b'\\' if in_single && in_escape_single => {
index = (index + 2).min(bytes.len());
continue;
}
b'\'' if !in_double => {
if in_single && index + 1 < bytes.len() && bytes[index + 1] == b'\'' {
index += 2;
continue;
}
if !in_single {
in_escape_single = is_escape_string_quote(bytes, index);
}
in_single = !in_single;
if !in_single {
in_escape_single = false;
}
}
b'"' if !in_single => {
if in_double && index + 1 < bytes.len() && bytes[index + 1] == b'"' {
index += 2;
continue;
}
in_double = !in_double;
}
b'-' if !in_single && !in_double && bytes.get(index + 1) == Some(&b'-') => {
while index < bytes.len() && bytes[index] != b'\n' {
index += 1;
}
continue;
}
b'/' if !in_single && !in_double && bytes.get(index + 1) == Some(&b'*') => {
index += 2;
while index < bytes.len() && !(bytes[index] == b'*' && bytes.get(index + 1) == Some(&b'/')) {
index += 1;
}
index = (index + 2).min(bytes.len());
continue;
}
b'$' if !in_single && !in_double => {
if let Some(end) = dollar_quote_end(bytes, index) {
index = end;
continue;
}
}
b';' if !in_single && !in_double => {
push_statement(&mut statements, &script[start..index]);
start = index + 1;
}
_ => {}
}
index += 1;
}
push_statement(&mut statements, &script[start..]);
statements
}
fn is_escape_string_quote(bytes: &[u8], quote_index: usize) -> bool {
if quote_index == 0 || !matches!(bytes[quote_index - 1], b'E' | b'e') {
return false;
}
quote_index == 1 || !(bytes[quote_index - 2].is_ascii_alphanumeric() || bytes[quote_index - 2] == b'_')
}
fn push_statement(statements: &mut Vec<String>, fragment: &str) {
let trimmed = fragment.trim();
if !trimmed.is_empty() {
statements.push(trimmed.to_string());
}
}
fn dollar_quote_end(bytes: &[u8], start: usize) -> Option<usize> {
let mut tag_end = start + 1;
while tag_end < bytes.len() && (bytes[tag_end] == b'_' || bytes[tag_end].is_ascii_alphanumeric()) {
tag_end += 1;
}
if tag_end >= bytes.len() || bytes[tag_end] != b'$' {
return None;
}
if tag_end > start + 1 && bytes[start + 1].is_ascii_digit() {
return None;
}
let tag = &bytes[start..=tag_end];
let mut index = tag_end + 1;
while index + tag.len() <= bytes.len() {
if &bytes[index..index + tag.len()] == tag {
return Some(index + tag.len());
}
index += 1;
}
Some(bytes.len())
}
pub fn attached_name_from_attach_sql(sql: &str) -> Option<String> {
let trimmed = strip_leading_comments(sql);
let first_word = trimmed.split(|character: char| character.is_whitespace() || character == ';').next()?;
if !first_word.eq_ignore_ascii_case("ATTACH") {
return None;
}
let as_index = find_as_keyword_outside_quotes(trimmed)?;
parse_identifier_after_as(&trimmed[as_index + 2..])
}
pub fn strip_leading_comments(sql: &str) -> &str {
let mut rest = sql.trim_start();
loop {
if let Some(after) = rest.strip_prefix("--") {
rest = after.split_once('\n').map(|(_, tail)| tail).unwrap_or("").trim_start();
} else if let Some(after) = rest.strip_prefix("/*") {
rest = after.split_once("*/").map(|(_, tail)| tail).unwrap_or("").trim_start();
} else {
return rest;
}
}
}
fn find_as_keyword_outside_quotes(sql: &str) -> Option<usize> {
let bytes = sql.as_bytes();
let mut index = 0;
let mut in_single = false;
let mut in_double = false;
while index < bytes.len() {
match bytes[index] {
b'\'' if !in_double => {
if in_single && index + 1 < bytes.len() && bytes[index + 1] == b'\'' {
index += 2;
continue;
}
in_single = !in_single;
}
b'"' if !in_single => {
if in_double && index + 1 < bytes.len() && bytes[index + 1] == b'"' {
index += 2;
continue;
}
in_double = !in_double;
}
b'a' | b'A'
if !in_single
&& !in_double
&& index + 1 < bytes.len()
&& matches!(bytes[index + 1], b's' | b'S')
&& is_sql_word_boundary(bytes.get(index.wrapping_sub(1)).copied())
&& is_sql_word_boundary(bytes.get(index + 2).copied()) =>
{
return Some(index);
}
_ => {}
}
index += 1;
}
None
}
fn is_sql_word_boundary(byte: Option<u8>) -> bool {
!matches!(byte, Some(b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'_'))
}
fn parse_identifier_after_as(input: &str) -> Option<String> {
let input = input.trim_start();
if input.is_empty() {
return None;
}
if let Some(rest) = input.strip_prefix('"') {
let mut name = String::new();
let mut characters = rest.chars().peekable();
while let Some(character) = characters.next() {
if character == '"' {
if characters.peek() == Some(&'"') {
name.push('"');
characters.next();
continue;
}
return (!name.trim().is_empty()).then_some(name);
}
name.push(character);
}
return None;
}
let name =
input.split(|character: char| character.is_whitespace() || character == ';').next().unwrap_or_default().trim();
(!name.is_empty()).then(|| name.to_string())
}
pub fn starts_with_duckdb_result_sql_keyword(sql: &str) -> bool {
let Some(token) = first_executable_sql_token(sql) else {
return false;
};
["SELECT", "SHOW", "DESCRIBE", "EXPLAIN", "WITH", "PRAGMA", "FROM", "SUMMARIZE", "SUMMARISE", "PIVOT", "UNPIVOT"]
.iter()
.any(|keyword| {
token.eq_ignore_ascii_case(keyword) || (*keyword == "DESCRIBE" && token.eq_ignore_ascii_case("DESC"))
})
}
fn first_executable_sql_token(sql: &str) -> Option<&str> {
let bytes = sql.as_bytes();
let mut index = 0;
while index < bytes.len() {
while index < bytes.len() && (bytes[index].is_ascii_whitespace() || bytes[index] == b'(') {
index += 1;
}
if index >= bytes.len() {
break;
}
if index + 1 < bytes.len() && bytes[index] == b'-' && bytes[index + 1] == b'-' {
index += 2;
while index < bytes.len() && bytes[index] != b'\n' {
index += 1;
}
continue;
}
if index + 1 < bytes.len() && bytes[index] == b'/' && bytes[index + 1] == b'*' {
index += 2;
while index + 1 < bytes.len() && !(bytes[index] == b'*' && bytes[index + 1] == b'/') {
index += 1;
}
index = (index + 2).min(bytes.len());
continue;
}
break;
}
let start = index;
while index < bytes.len() && (bytes[index].is_ascii_alphabetic() || bytes[index] == b'_') {
index += 1;
}
(index > start).then_some(&sql[start..index])
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn splits_quoted_and_commented_statements() {
let script = "SET s3_endpoint='host;port'; -- comment;\nATTACH 'db;x.duckdb' AS a; SELECT $$a;b$$";
assert_eq!(
split_sql_statements(script),
vec![
"SET s3_endpoint='host;port'".to_string(),
"-- comment;\nATTACH 'db;x.duckdb' AS a".to_string(),
"SELECT $$a;b$$".to_string(),
]
);
}
#[test]
fn detects_attach_alias_after_comments() {
assert_eq!(
attached_name_from_attach_sql("-- note\nATTACH 'sales.duckdb' AS \"sales db\";"),
Some("sales db".to_string())
);
}
#[test]
fn detects_result_statements_after_comments() {
assert!(starts_with_duckdb_result_sql_keyword("/* note */ WITH rows AS (SELECT 1) SELECT * FROM rows"));
assert!(starts_with_duckdb_result_sql_keyword("DESC SELECT 1"));
assert!(!starts_with_duckdb_result_sql_keyword("INSERT INTO items VALUES (1)"));
}
}
+302
View File
@@ -0,0 +1,302 @@
use serde::{de::DeserializeOwned, Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DatabaseInfo {
pub name: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TableInfo {
pub name: String,
pub table_type: String,
pub comment: Option<String>,
pub parent_schema: Option<String>,
pub parent_name: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub enum ObjectSourceKind {
View,
MaterializedView,
Procedure,
Function,
Trigger,
Sequence,
Package,
PackageBody,
Type,
TypeBody,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct ColumnInfo {
pub name: String,
pub data_type: String,
pub is_nullable: bool,
pub column_default: Option<String>,
pub is_primary_key: bool,
pub extra: Option<String>,
pub comment: Option<String>,
pub numeric_precision: Option<i32>,
pub numeric_scale: Option<i32>,
pub character_maximum_length: Option<i32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub enum_values: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub character_set: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub collation: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum CompletionAssistantObjectKind {
Database,
Schema,
Table,
View,
Routine,
Procedure,
Function,
Column,
}
impl CompletionAssistantObjectKind {
pub fn is_table_like(&self) -> bool {
matches!(self, Self::Table | Self::View)
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum CompletionAssistantCandidateKind {
Database,
Schema,
Table,
View,
Procedure,
Function,
Column,
Object,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum CompletionAssistantMatchMode {
Prefix,
Contains,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CompletionAssistantRequest {
pub connection_id: String,
pub database: String,
pub schema: Option<String>,
#[serde(default)]
pub object_kinds: Vec<CompletionAssistantObjectKind>,
#[serde(default)]
pub mask: String,
#[serde(default)]
pub case_sensitive: bool,
#[serde(default)]
pub global_search: bool,
pub max_results: Option<usize>,
#[serde(default)]
pub search_in_comments: bool,
#[serde(default)]
pub search_in_definitions: bool,
pub parent_schema: Option<String>,
pub parent_name: Option<String>,
pub match_mode: Option<CompletionAssistantMatchMode>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CompletionAssistantCandidate {
pub name: String,
pub kind: CompletionAssistantCandidateKind,
pub database: Option<String>,
pub schema: Option<String>,
pub parent_schema: Option<String>,
pub parent_name: Option<String>,
pub comment: Option<String>,
pub data_type: Option<String>,
pub signature: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CompletionAssistantResponse {
pub candidates: Vec<CompletionAssistantCandidate>,
pub incomplete: bool,
pub fallback_used: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct QueryResult {
pub columns: Vec<String>,
#[serde(default)]
pub column_types: Vec<String>,
#[serde(default)]
pub column_sortables: Vec<bool>,
pub rows: Vec<Vec<serde_json::Value>>,
pub affected_rows: u64,
pub execution_time_ms: u128,
#[serde(default)]
pub truncated: bool,
#[serde(default)]
pub session_id: Option<String>,
#[serde(default)]
pub has_more: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub elasticsearch_raw_body: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct AttachedDatabaseConfig {
pub name: String,
pub path: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct DuckDbWorkerRequest {
pub id: String,
pub method: DuckDbWorkerMethod,
#[serde(default)]
pub params: serde_json::Value,
}
impl DuckDbWorkerRequest {
#[cfg(test)]
pub fn new(id: impl Into<String>, method: DuckDbWorkerMethod, params: impl Serialize) -> Result<Self, String> {
let params = serde_json::to_value(params).map_err(|error| error.to_string())?;
Ok(Self { id: id.into(), method, params })
}
pub fn parse_params<T: DeserializeOwned>(&self) -> Result<T, DuckDbWorkerError> {
serde_json::from_value(self.params.clone()).map_err(|error| {
DuckDbWorkerError::new("invalid_params", format!("Invalid params for {:?}: {error}", self.method))
})
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum DuckDbWorkerMethod {
Connect,
Execute,
ListDatabases,
ListSchemas,
ListTables,
ListColumns,
GetTableDdl,
CompletionAssistant,
GetObjectSource,
AttachDatabase,
Cancel,
Shutdown,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct DuckDbWorkerResponse {
pub id: String,
pub ok: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub result: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<DuckDbWorkerError>,
}
impl DuckDbWorkerResponse {
pub fn ok(id: impl Into<String>, result: impl Serialize) -> Self {
match serde_json::to_value(result) {
Ok(result) => Self { id: id.into(), ok: true, result: Some(result), error: None },
Err(error) => Self::err(id, DuckDbWorkerError::new("serialization_failed", error.to_string())),
}
}
pub fn ok_empty(id: impl Into<String>) -> Self {
Self { id: id.into(), ok: true, result: None, error: None }
}
pub fn err(id: impl Into<String>, error: DuckDbWorkerError) -> Self {
Self { id: id.into(), ok: false, result: None, error: Some(error) }
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct DuckDbWorkerError {
pub code: String,
pub message: String,
}
impl DuckDbWorkerError {
pub fn new(code: impl Into<String>, message: impl Into<String>) -> Self {
Self { code: code.into(), message: message.into() }
}
pub fn from_message(code: &'static str, error: impl ToString) -> Self {
Self::new(code, error.to_string())
}
}
impl From<String> for DuckDbWorkerError {
fn from(message: String) -> Self {
Self::new("duckdb_worker_error", message)
}
}
impl From<&str> for DuckDbWorkerError {
fn from(message: &str) -> Self {
Self::new("duckdb_worker_error", message)
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct DuckDbWorkerConnectParams {
pub path: String,
#[serde(default)]
pub attached_databases: Vec<AttachedDatabaseConfig>,
#[serde(default)]
pub init_script: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct DuckDbWorkerExecuteParams {
pub sql: String,
#[serde(default)]
pub database: Option<String>,
#[serde(default)]
pub max_rows: Option<usize>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct DuckDbWorkerDatabaseParams {
pub database: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct DuckDbWorkerTableParams {
pub database: String,
pub schema: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct DuckDbWorkerColumnParams {
pub database: String,
pub schema: String,
pub table: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct DuckDbWorkerObjectSourceParams {
pub database: String,
pub schema: String,
pub name: String,
pub object_type: ObjectSourceKind,
}
@@ -1,5 +1,3 @@
#![cfg(feature = "duckdb-bundled")]
use std::path::PathBuf; use std::path::PathBuf;
use std::process::Stdio; use std::process::Stdio;
use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::atomic::{AtomicU64, Ordering};
@@ -21,7 +19,7 @@ static TEMP_DB_COUNTER: AtomicU64 = AtomicU64::new(0);
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn worker_process_reads_view_source() { async fn worker_process_reads_view_source() {
let _guard = duckdb_worker_process_test_guard().await; let _guard = duckdb_worker_process_test_guard().await;
let executable = PathBuf::from(env!("CARGO_BIN_EXE_duckdb-worker-test-host")); let executable = PathBuf::from(env!("CARGO_BIN_EXE_dbx-duckdb-driver"));
let db_path = temp_duckdb_path(); let db_path = temp_duckdb_path();
let _ = std::fs::remove_file(&db_path); let _ = std::fs::remove_file(&db_path);
@@ -57,10 +55,45 @@ async fn worker_process_reads_view_source() {
let _ = std::fs::remove_file(&db_path); let _ = std::fs::remove_file(&db_path);
} }
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn worker_process_reads_table_ddl() {
let _guard = duckdb_worker_process_test_guard().await;
let executable = PathBuf::from(env!("CARGO_BIN_EXE_dbx-duckdb-driver"));
let db_path = temp_duckdb_path();
let _ = std::fs::remove_file(&db_path);
let client =
DuckDbWorkerClient::open_with_executable(executable, db_path.to_string_lossy().to_string(), Vec::new(), None)
.await
.expect("worker process connects");
client
.execute(
None,
"CREATE TABLE orders(order_id BIGINT, amount DECIMAL(18, 2))".to_string(),
None,
None,
Some(Duration::from_secs(5)),
)
.await
.expect("create table");
let ddl = client
.get_table_ddl("main".to_string(), "main".to_string(), "orders".to_string())
.await
.expect("get table ddl");
assert!(ddl.starts_with("CREATE TABLE"));
assert!(ddl.contains("order_id"));
assert!(ddl.contains("DECIMAL(18,2)"));
client.shutdown().await;
let _ = std::fs::remove_file(&db_path);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn worker_process_recovers_immediately_after_cancelled_long_query() { async fn worker_process_recovers_immediately_after_cancelled_long_query() {
let _guard = duckdb_worker_process_test_guard().await; let _guard = duckdb_worker_process_test_guard().await;
let executable = PathBuf::from(env!("CARGO_BIN_EXE_duckdb-worker-test-host")); let executable = PathBuf::from(env!("CARGO_BIN_EXE_dbx-duckdb-driver"));
let db_path = temp_duckdb_path(); let db_path = temp_duckdb_path();
let _ = std::fs::remove_file(&db_path); let _ = std::fs::remove_file(&db_path);
@@ -111,7 +144,7 @@ async fn worker_process_recovers_immediately_after_cancelled_long_query() {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn worker_process_recovers_after_registered_cancel_interrupt() { async fn worker_process_recovers_after_registered_cancel_interrupt() {
let _guard = duckdb_worker_process_test_guard().await; let _guard = duckdb_worker_process_test_guard().await;
let executable = PathBuf::from(env!("CARGO_BIN_EXE_duckdb-worker-test-host")); let executable = PathBuf::from(env!("CARGO_BIN_EXE_dbx-duckdb-driver"));
let db_path = temp_duckdb_path(); let db_path = temp_duckdb_path();
let _ = std::fs::remove_file(&db_path); let _ = std::fs::remove_file(&db_path);
@@ -174,7 +207,7 @@ async fn worker_process_recovers_after_registered_cancel_interrupt() {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn worker_process_recovers_after_parser_error() { async fn worker_process_recovers_after_parser_error() {
let _guard = duckdb_worker_process_test_guard().await; let _guard = duckdb_worker_process_test_guard().await;
let executable = PathBuf::from(env!("CARGO_BIN_EXE_duckdb-worker-test-host")); let executable = PathBuf::from(env!("CARGO_BIN_EXE_dbx-duckdb-driver"));
let db_path = temp_duckdb_path(); let db_path = temp_duckdb_path();
let _ = std::fs::remove_file(&db_path); let _ = std::fs::remove_file(&db_path);
@@ -224,7 +257,7 @@ async fn worker_process_recovers_after_parser_error() {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn worker_process_keeps_session_state_after_benign_error() { async fn worker_process_keeps_session_state_after_benign_error() {
let _guard = duckdb_worker_process_test_guard().await; let _guard = duckdb_worker_process_test_guard().await;
let executable = PathBuf::from(env!("CARGO_BIN_EXE_duckdb-worker-test-host")); let executable = PathBuf::from(env!("CARGO_BIN_EXE_dbx-duckdb-driver"));
let db_path = temp_duckdb_path(); let db_path = temp_duckdb_path();
let _ = std::fs::remove_file(&db_path); let _ = std::fs::remove_file(&db_path);
@@ -272,7 +305,7 @@ async fn worker_process_keeps_session_state_after_benign_error() {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn worker_process_exits_when_stdin_closes_during_active_query() { async fn worker_process_exits_when_stdin_closes_during_active_query() {
let _guard = duckdb_worker_process_test_guard().await; let _guard = duckdb_worker_process_test_guard().await;
let executable = PathBuf::from(env!("CARGO_BIN_EXE_duckdb-worker-test-host")); let executable = PathBuf::from(env!("CARGO_BIN_EXE_dbx-duckdb-driver"));
let db_path = temp_duckdb_path(); let db_path = temp_duckdb_path();
let _ = std::fs::remove_file(&db_path); let _ = std::fs::remove_file(&db_path);
let mut child = Command::new(executable) let mut child = Command::new(executable)
@@ -339,6 +372,7 @@ async fn worker_process_is_killed_after_connect_timeout() {
std::env::set_var("DBX_DUCKDB_HANGING_CONNECT_PID_FILE", &pid_file); std::env::set_var("DBX_DUCKDB_HANGING_CONNECT_PID_FILE", &pid_file);
let client = DuckDbWorkerClient::new_unconnected_with_timeouts( let client = DuckDbWorkerClient::new_unconnected_with_timeouts(
executable, executable,
Vec::new(),
db_path.to_string_lossy().to_string(), db_path.to_string_lossy().to_string(),
Vec::new(), Vec::new(),
None, None,
@@ -374,6 +408,7 @@ async fn worker_process_is_killed_after_connect_error() {
std::env::set_var("DBX_DUCKDB_PID_TEST_HOST_PID_FILE", &pid_file); std::env::set_var("DBX_DUCKDB_PID_TEST_HOST_PID_FILE", &pid_file);
let client = DuckDbWorkerClient::new_unconnected_with_timeouts( let client = DuckDbWorkerClient::new_unconnected_with_timeouts(
executable, executable,
Vec::new(),
db_path.to_string_lossy().to_string(), db_path.to_string_lossy().to_string(),
Vec::new(), Vec::new(),
None, None,
@@ -405,7 +440,7 @@ async fn worker_process_is_killed_after_connect_error() {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn worker_process_retries_connect_after_transient_file_lock() { async fn worker_process_retries_connect_after_transient_file_lock() {
let _guard = duckdb_worker_process_test_guard().await; let _guard = duckdb_worker_process_test_guard().await;
let executable = PathBuf::from(env!("CARGO_BIN_EXE_duckdb-worker-test-host")); let executable = PathBuf::from(env!("CARGO_BIN_EXE_dbx-duckdb-driver"));
let lock_owner_executable = PathBuf::from(env!("CARGO_BIN_EXE_duckdb-worker-file-lock-owner")); let lock_owner_executable = PathBuf::from(env!("CARGO_BIN_EXE_duckdb-worker-file-lock-owner"));
let db_path = temp_duckdb_path(); let db_path = temp_duckdb_path();
let _ = std::fs::remove_file(&db_path); let _ = std::fs::remove_file(&db_path);
@@ -429,6 +464,7 @@ async fn worker_process_retries_connect_after_transient_file_lock() {
let client = DuckDbWorkerClient::new_unconnected_with_timeouts( let client = DuckDbWorkerClient::new_unconnected_with_timeouts(
executable, executable,
Vec::new(),
db_path.to_string_lossy().to_string(), db_path.to_string_lossy().to_string(),
Vec::new(), Vec::new(),
None, None,
@@ -1,5 +1,3 @@
#![cfg(feature = "duckdb-bundled")]
use std::io::Write; use std::io::Write;
use std::path::PathBuf; use std::path::PathBuf;
use std::time::Duration; use std::time::Duration;
@@ -1,5 +1,3 @@
#![cfg(feature = "duckdb-bundled")]
use std::io::{self, BufRead}; use std::io::{self, BufRead};
use std::time::Duration; use std::time::Duration;
@@ -1,12 +1,10 @@
#![cfg(feature = "duckdb-bundled")]
fn main() { fn main() {
if let Ok(path) = std::env::var("DBX_DUCKDB_PID_TEST_HOST_PID_FILE") { if let Ok(path) = std::env::var("DBX_DUCKDB_PID_TEST_HOST_PID_FILE") {
let _ = std::fs::write(path, std::process::id().to_string()); let _ = std::fs::write(path, std::process::id().to_string());
} }
let runtime = tokio::runtime::Runtime::new().expect("Failed to create DuckDB worker test runtime"); let runtime = tokio::runtime::Runtime::new().expect("Failed to create DuckDB worker test runtime");
if let Err(err) = runtime.block_on(dbx_core::db::duckdb_worker_runtime::run_stdio_worker()) { if let Err(err) = runtime.block_on(dbx_duckdb_driver::run_stdio_worker()) {
eprintln!("DuckDB worker test host failed: {err}"); eprintln!("DuckDB worker test host failed: {err}");
std::process::exit(1); std::process::exit(1);
} }
+74 -12
View File
@@ -1,8 +1,12 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import argparse import argparse
import copy import copy
import hashlib
import io
import json import json
import zipfile import subprocess
import tarfile
import tempfile
from pathlib import Path from pathlib import Path
from urllib.parse import urlparse from urllib.parse import urlparse
@@ -11,10 +15,51 @@ def artifact_filename(url: str) -> str:
return Path(urlparse(url).path).name return Path(urlparse(url).path).name
def write_driver_zip(output: Path, registry: dict, source: Path) -> None: def write_driver_tar_zstd(output: Path, registry: dict, source: Path, *, executable: bool) -> None:
with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive: registry_bytes = (json.dumps(registry, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
archive.writestr("agent-registry.json", json.dumps(registry, ensure_ascii=False, indent=2) + "\n") with tempfile.TemporaryDirectory() as temp_dir:
archive.write(source, f"drivers/{source.name}") tar_path = Path(temp_dir) / "driver.tar"
with tarfile.open(tar_path, "w", format=tarfile.PAX_FORMAT) as archive:
registry_info = tarfile.TarInfo("agent-registry.json")
registry_info.size = len(registry_bytes)
registry_info.mode = 0o644
registry_info.mtime = 0
archive.addfile(registry_info, io.BytesIO(registry_bytes))
driver_info = archive.gettarinfo(str(source), arcname=f"drivers/{source.name}")
driver_info.mode = 0o755 if executable else 0o644
driver_info.mtime = 0
driver_info.uid = 0
driver_info.gid = 0
driver_info.uname = ""
driver_info.gname = ""
with source.open("rb") as driver_file:
archive.addfile(driver_info, driver_file)
subprocess.run(
["zstd", "-q", "-19", "--force", str(tar_path), "-o", str(output)],
check=True,
)
def release_url_with_filename(url: str, filename: str) -> str:
prefix, separator, _ = url.rpartition("/")
return f"{prefix}{separator}{filename}" if separator else filename
def packaged_artifact(artifact: dict, source: Path) -> dict:
packaged = copy.deepcopy(artifact)
packaged["url"] = source.name
packaged["size"] = source.stat().st_size
packaged.pop("format", None)
return packaged
def update_release_artifact(artifact: dict, output: Path) -> None:
artifact["url"] = release_url_with_filename(artifact["url"], output.name)
artifact["size"] = output.stat().st_size
artifact["sha256"] = hashlib.sha256(output.read_bytes()).hexdigest()
artifact["format"] = "tar_zstd"
def build_driver_zips(release_dir: Path) -> list[Path]: def build_driver_zips(release_dir: Path) -> list[Path]:
@@ -33,10 +78,11 @@ def build_driver_zips(release_dir: Path) -> list[Path]:
package_driver = copy.deepcopy(driver) package_driver = copy.deepcopy(driver)
package_driver.pop("native", None) package_driver.pop("native", None)
package_driver["jar"] = {"url": source.name, "size": source.stat().st_size} package_driver["jar"] = packaged_artifact(jar_artifact, source)
package_registry = {"jres": {}, "drivers": {driver_name: package_driver}} package_registry = {"jres": {}, "drivers": {driver_name: package_driver}}
output = release_dir / f"dbx-agent-{driver_name}-{version}.zip" output = release_dir / f"dbx-agent-{driver_name}-{version}.tar.zst"
write_driver_zip(output, package_registry, source) write_driver_tar_zstd(output, package_registry, source, executable=False)
update_release_artifact(jar_artifact, output)
outputs.append(output) outputs.append(output)
for platform, artifact in driver.get("native", {}).items(): for platform, artifact in driver.get("native", {}).items():
@@ -47,22 +93,38 @@ def build_driver_zips(release_dir: Path) -> list[Path]:
package_driver = copy.deepcopy(driver) package_driver = copy.deepcopy(driver)
package_driver.pop("jar", None) package_driver.pop("jar", None)
package_driver["native"] = {platform: {"url": source.name, "size": source.stat().st_size}} package_driver["native"] = {platform: packaged_artifact(artifact, source)}
package_registry = {"jres": {}, "drivers": {driver_name: package_driver}} package_registry = {"jres": {}, "drivers": {driver_name: package_driver}}
output = release_dir / f"dbx-agent-{driver_name}-{version}-{platform}.zip" output = release_dir / f"dbx-agent-{driver_name}-{version}-{platform}.tar.zst"
write_driver_zip(output, package_registry, source) write_driver_tar_zstd(output, package_registry, source, executable=True)
update_release_artifact(artifact, output)
outputs.append(output) outputs.append(output)
registry_path.write_text(json.dumps(registry, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return outputs return outputs
def remove_raw_driver_artifacts(release_dir: Path) -> list[Path]:
removed: list[Path] = []
for path in sorted(release_dir.glob("dbx-agent-*")):
if path.name.endswith(".tar.zst") or not path.is_file():
continue
path.unlink()
removed.append(path)
return removed
def main() -> None: def main() -> None:
parser = argparse.ArgumentParser(description="Build offline ZIPs for individual DBX agents") parser = argparse.ArgumentParser(description="Build tar.zst packages for individual DBX agents")
parser.add_argument("release_dir", type=Path) parser.add_argument("release_dir", type=Path)
parser.add_argument("--cleanup-sources", action="store_true")
args = parser.parse_args() args = parser.parse_args()
for path in build_driver_zips(args.release_dir): for path in build_driver_zips(args.release_dir):
print(f"Created {path.name} ({path.stat().st_size} bytes)") print(f"Created {path.name} ({path.stat().st_size} bytes)")
if args.cleanup_sources:
for path in remove_raw_driver_artifacts(args.release_dir):
print(f"Removed intermediate {path.name}")
if __name__ == "__main__": if __name__ == "__main__":
+2 -2
View File
@@ -3,7 +3,7 @@ set -euo pipefail
# Build offline ZIP bundles for each platform. # Build offline ZIP bundles for each platform.
# Usage: ./scripts/build_offline_zip.sh <release-dir> # Usage: ./scripts/build_offline_zip.sh <release-dir>
# The release-dir must contain agent-registry.json, dbx-agent-*.jar, and dbx-jre-*.tar.gz # The release-dir must contain agent-registry.json, raw driver artifacts, and dbx-jre-*.tar.zst
RELEASE_DIR="$(cd "${1:?Usage: build_offline_zip.sh <release-dir>}" && pwd)" RELEASE_DIR="$(cd "${1:?Usage: build_offline_zip.sh <release-dir>}" && pwd)"
@@ -32,7 +32,7 @@ for platform in "${PLATFORMS[@]}"; do
# Copy all JRE archives for this platform # Copy all JRE archives for this platform
JRE_COUNT=0 JRE_COUNT=0
for jre_file in "$RELEASE_DIR"/dbx-jre-*-"$platform".tar.gz; do for jre_file in "$RELEASE_DIR"/dbx-jre-*-"$platform".tar.zst; do
[ -f "$jre_file" ] || continue [ -f "$jre_file" ] || continue
cp "$jre_file" "$WORK/jre/" cp "$jre_file" "$WORK/jre/"
JRE_COUNT=$((JRE_COUNT + 1)) JRE_COUNT=$((JRE_COUNT + 1))
+74 -25
View File
@@ -1,29 +1,40 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import io
import json import json
import subprocess import subprocess
import tarfile
import tempfile import tempfile
import unittest import unittest
import zipfile import zipfile
from pathlib import Path from pathlib import Path
from build_driver_zips import build_driver_zips from build_driver_zips import build_driver_zips, remove_raw_driver_artifacts
from version_agent_artifacts import version_agent_artifacts from version_agent_artifacts import version_agent_artifacts
class DriverReleasePackagesTest(unittest.TestCase): class DriverReleasePackagesTest(unittest.TestCase):
def test_builds_java_and_platform_specific_native_driver_zips(self) -> None: def test_builds_java_and_platform_specific_native_driver_tar_zstd_packages(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir: with tempfile.TemporaryDirectory() as temp_dir:
release_dir = Path(temp_dir) release_dir = Path(temp_dir)
native_source = release_dir / "dbx-agent-kingbase-windows-x64.exe" native_source = release_dir / "dbx-agent-kingbase-windows-x64.exe"
native_source.write_bytes(b"MZtest-agent") native_source.write_bytes(b"MZtest-agent")
duckdb_source = release_dir / "dbx-agent-duckdb-macos-aarch64"
duckdb_source.write_bytes(b"\xcf\xfa\xed\xfetest-duckdb-agent")
java_source = release_dir / "dbx-agent-h2.jar" java_source = release_dir / "dbx-agent-h2.jar"
java_source.write_bytes(b"test-jar") java_source.write_bytes(b"test-jar")
versions = {"h2": "0.2.5", "oracle": "0.1.10", "xugu": "0.1.20", "kingbase": "0.1.34"} versions = {
"h2": "0.2.5",
"oracle": "0.1.10",
"xugu": "0.1.20",
"kingbase": "0.1.34",
"duckdb": "0.1.0",
}
renamed = version_agent_artifacts(release_dir, versions) renamed = version_agent_artifacts(release_dir, versions)
versioned_java = release_dir / "dbx-agent-h2-0.2.5.jar" versioned_java = release_dir / "dbx-agent-h2-0.2.5.jar"
versioned_native = release_dir / "dbx-agent-kingbase-0.1.34-windows-x64.exe" versioned_native = release_dir / "dbx-agent-kingbase-0.1.34-windows-x64.exe"
self.assertEqual(renamed, [versioned_java, versioned_native]) versioned_duckdb = release_dir / "dbx-agent-duckdb-0.1.0-macos-aarch64"
self.assertEqual(renamed, [versioned_java, versioned_native, versioned_duckdb])
registry = { registry = {
"jres": {"21": {"version": "21", "platforms": {}}}, "jres": {"21": {"version": "21", "platforms": {}}},
@@ -48,6 +59,19 @@ class DriverReleasePackagesTest(unittest.TestCase):
} }
}, },
}, },
"duckdb": {
"version": "0.1.0",
"label": "DuckDB",
"min_app_version": "0.6.0",
"jre": "21",
"jar": {"url": "https://example.com/legacy-placeholder.jar", "size": 0},
"native": {
"macos-aarch64": {
"url": f"https://example.com/{versioned_duckdb.name}",
"size": versioned_duckdb.stat().st_size,
}
},
},
}, },
} }
(release_dir / "agent-registry.json").write_text(json.dumps(registry), encoding="utf-8") (release_dir / "agent-registry.json").write_text(json.dumps(registry), encoding="utf-8")
@@ -57,26 +81,51 @@ class DriverReleasePackagesTest(unittest.TestCase):
self.assertEqual( self.assertEqual(
outputs, outputs,
[ [
release_dir / "dbx-agent-h2-0.2.5.zip", release_dir / "dbx-agent-h2-0.2.5.tar.zst",
release_dir / "dbx-agent-kingbase-0.1.34-windows-x64.zip", release_dir / "dbx-agent-kingbase-0.1.34-windows-x64.tar.zst",
release_dir / "dbx-agent-duckdb-0.1.0-macos-aarch64.tar.zst",
], ],
) )
with zipfile.ZipFile(outputs[0]) as archive: package_cases = [
self.assertEqual(set(archive.namelist()), {"agent-registry.json", f"drivers/{versioned_java.name}"}) (outputs[0], "h2", versioned_java, "jar", None),
package_registry = json.loads(archive.read("agent-registry.json")) (outputs[1], "kingbase", versioned_native, "native", "windows-x64"),
self.assertEqual(set(package_registry["drivers"]), {"h2"}) (outputs[2], "duckdb", versioned_duckdb, "native", "macos-aarch64"),
self.assertNotIn("native", package_registry["drivers"]["h2"]) ]
self.assertEqual(package_registry["drivers"]["h2"]["jar"], {"url": versioned_java.name, "size": 8}) for output, driver_name, source, artifact_type, platform in package_cases:
with zipfile.ZipFile(outputs[1]) as archive: tar_bytes = subprocess.run(
self.assertEqual(set(archive.namelist()), {"agent-registry.json", f"drivers/{versioned_native.name}"}) ["zstd", "-q", "-dc", str(output)],
package_registry = json.loads(archive.read("agent-registry.json")) check=True,
kingbase = package_registry["drivers"]["kingbase"] capture_output=True,
self.assertNotIn("jar", kingbase) ).stdout
self.assertEqual(set(kingbase["native"]), {"windows-x64"}) with tarfile.open(fileobj=io.BytesIO(tar_bytes), mode="r:") as archive:
self.assertEqual( self.assertEqual(set(archive.getnames()), {"agent-registry.json", f"drivers/{source.name}"})
kingbase["native"]["windows-x64"], package_registry = json.load(archive.extractfile("agent-registry.json"))
{"url": versioned_native.name, "size": versioned_native.stat().st_size}, driver = package_registry["drivers"][driver_name]
) if artifact_type == "jar":
self.assertNotIn("native", driver)
self.assertEqual(driver["jar"], {"url": source.name, "size": source.stat().st_size})
else:
self.assertNotIn("jar", driver)
self.assertEqual(
driver["native"][platform],
{"url": source.name, "size": source.stat().st_size},
)
final_registry = json.loads((release_dir / "agent-registry.json").read_text(encoding="utf-8"))
release_artifacts = [
(final_registry["drivers"]["h2"]["jar"], outputs[0]),
(final_registry["drivers"]["kingbase"]["native"]["windows-x64"], outputs[1]),
(final_registry["drivers"]["duckdb"]["native"]["macos-aarch64"], outputs[2]),
]
for artifact, output in release_artifacts:
self.assertEqual(artifact["url"], f"https://example.com/{output.name}")
self.assertEqual(artifact["size"], output.stat().st_size)
self.assertEqual(artifact["format"], "tar_zstd")
self.assertEqual(len(artifact["sha256"]), 64)
removed = remove_raw_driver_artifacts(release_dir)
self.assertEqual(removed, [versioned_duckdb, versioned_java, versioned_native])
self.assertTrue(all(output.is_file() for output in outputs))
def test_full_offline_bundle_includes_supported_windows_artifacts(self) -> None: def test_full_offline_bundle_includes_supported_windows_artifacts(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir: with tempfile.TemporaryDirectory() as temp_dir:
@@ -85,8 +134,8 @@ class DriverReleasePackagesTest(unittest.TestCase):
kafka_filename = "dbx-agent-kafka-0.1.0.jar" kafka_filename = "dbx-agent-kafka-0.1.0.jar"
(release_dir / filename).write_bytes(b"MZtest-agent") (release_dir / filename).write_bytes(b"MZtest-agent")
(release_dir / kafka_filename).write_bytes(b"test-kafka-agent") (release_dir / kafka_filename).write_bytes(b"test-kafka-agent")
(release_dir / "dbx-jre-21-windows-x64.tar.gz").write_bytes(b"test-jre") (release_dir / "dbx-jre-21-windows-x64.tar.zst").write_bytes(b"test-jre")
(release_dir / "dbx-jre-21-windows-aarch64.tar.gz").write_bytes(b"test-jre") (release_dir / "dbx-jre-21-windows-aarch64.tar.zst").write_bytes(b"test-jre")
(release_dir / "agent-registry.json").write_text('{"jres":{},"drivers":{}}', encoding="utf-8") (release_dir / "agent-registry.json").write_text('{"jres":{},"drivers":{}}', encoding="utf-8")
result = subprocess.run( result = subprocess.run(
@@ -105,7 +154,7 @@ class DriverReleasePackagesTest(unittest.TestCase):
self.assertIn(f"drivers/{filename}", archive.namelist()) self.assertIn(f"drivers/{filename}", archive.namelist())
self.assertIn(f"drivers/{kafka_filename}", archive.namelist()) self.assertIn(f"drivers/{kafka_filename}", archive.namelist())
with zipfile.ZipFile(arm64_bundle) as archive: with zipfile.ZipFile(arm64_bundle) as archive:
self.assertIn("jre/dbx-jre-21-windows-aarch64.tar.gz", archive.namelist()) self.assertIn("jre/dbx-jre-21-windows-aarch64.tar.zst", archive.namelist())
self.assertIn(f"drivers/{kafka_filename}", archive.namelist()) self.assertIn(f"drivers/{kafka_filename}", archive.namelist())
+6 -1
View File
@@ -13,10 +13,12 @@ KOTLIN_SCAN_EXCLUDED_PARTS = {".git", ".gradle", "build"}
DEFAULT_AGENT_JRE_KEY = "21" DEFAULT_AGENT_JRE_KEY = "21"
NON_JDBC_AGENT_MODULES = {"mongodb", "etcd", "zookeeper", "kafka", "rocketmq", "rabbitmq"} NON_JDBC_AGENT_MODULES = {"mongodb", "etcd", "zookeeper", "kafka", "rocketmq", "rabbitmq"}
NATIVE_ONLY_AGENT_MODULES = { NATIVE_ONLY_AGENT_MODULES = {
"duckdb": "drivers/duckdb",
"oracle": "drivers/oracle-go", "oracle": "drivers/oracle-go",
"kingbase": "drivers/kingbase-go", "kingbase": "drivers/kingbase-go",
"xugu": "drivers/xugu", "xugu": "drivers/xugu",
} }
AUTO_VERSIONED_NATIVE_MODULES = {"duckdb"}
JDBC_ARCHITECTURE_ALLOWLIST = { JDBC_ARCHITECTURE_ALLOWLIST = {
"h2-legacy": "reuses the H2 agent implementation with an isolated legacy driver version", "h2-legacy": "reuses the H2 agent implementation with an isolated legacy driver version",
"access": "custom Access metadata and URL behavior pending migration", "access": "custom Access metadata and URL behavior pending migration",
@@ -90,7 +92,10 @@ def validate_versions(root: Path) -> list[str]:
included = agent_modules(root) included = agent_modules(root)
versions = set(json.loads((root / "versions.json").read_text(encoding="utf-8"))) versions = set(json.loads((root / "versions.json").read_text(encoding="utf-8")))
return ( return (
[f"included module missing version: {name}" for name in sorted(included - versions)] [
f"included module missing version: {name}"
for name in sorted(included - versions - AUTO_VERSIONED_NATIVE_MODULES)
]
+ [f"versions key not included: {name}" for name in sorted(versions - included)] + [f"versions key not included: {name}" for name in sorted(versions - included)]
) )
+1 -1
View File
@@ -136,7 +136,7 @@ class ValidateAgentsTest(unittest.TestCase):
"include(*(infrastructureModules + driverModules))\n", "include(*(infrastructureModules + driverModules))\n",
encoding="utf-8", encoding="utf-8",
) )
for driver in ("oracle-go", "kingbase-go", "xugu"): for driver in ("oracle-go", "kingbase-go", "xugu", "duckdb"):
(root / "drivers" / driver).mkdir(parents=True) (root / "drivers" / driver).mkdir(parents=True)
(root / "versions.json").write_text( (root / "versions.json").write_text(
json.dumps({"h2": "0.1.0", "oracle": "0.1.0", "kingbase": "0.1.0", "xugu": "0.1.0"}), json.dumps({"h2": "0.1.0", "oracle": "0.1.0", "kingbase": "0.1.0", "xugu": "0.1.0"}),
+1 -1
View File
@@ -4,7 +4,7 @@ import json
from pathlib import Path from pathlib import Path
NATIVE_DRIVERS = ("oracle", "xugu", "kingbase") NATIVE_DRIVERS = ("oracle", "xugu", "kingbase", "duckdb")
PLATFORMS = ( PLATFORMS = (
"macos-aarch64", "macos-aarch64",
"macos-x64", "macos-x64",
+1
View File
@@ -6,6 +6,7 @@
"databend": "0.1.22", "databend": "0.1.22",
"databricks": "0.1.29", "databricks": "0.1.29",
"db2": "0.1.34", "db2": "0.1.34",
"duckdb": "0.1.0",
"exasol": "0.1.29", "exasol": "0.1.29",
"firebird": "0.1.29", "firebird": "0.1.29",
"gbase8a": "0.1.29", "gbase8a": "0.1.29",
@@ -552,6 +552,11 @@ async function uninstallDriver(dbType: string) {
toast(t("driverStore.driverUninstallSuccess", { label })); toast(t("driverStore.driverUninstallSuccess", { label }));
return; return;
} }
const blockers = await api.checkAgentUpdateBlockers([dbType]);
if (blockers.length > 0) {
toast(t("driverStore.driverUpdateBlocked", { labels: blockers.map((blocker) => blocker.label).join(", ") }));
return;
}
await api.uninstallAgent(dbType); await api.uninstallAgent(dbType);
await refreshAgents(); await refreshAgents();
toast(t("driverStore.driverUninstallSuccess", { label })); toast(t("driverStore.driverUninstallSuccess", { label }));
@@ -561,12 +566,14 @@ async function uninstallDriver(dbType: string) {
} }
const importingZip = ref(false); const importingZip = ref(false);
const importingDriver = ref<string | null>(null);
const agentImportBusy = computed(() => importingZip.value || importingDriver.value !== null);
function chooseWebOfflineZip(): Promise<File | null> { function chooseWebOfflineZip(): Promise<File | null> {
return new Promise((resolve) => { return new Promise((resolve) => {
const input = document.createElement("input"); const input = document.createElement("input");
input.type = "file"; input.type = "file";
input.accept = ".zip"; input.accept = ".zip,.tar.zst";
input.onchange = () => resolve(input.files?.[0] ?? null); input.onchange = () => resolve(input.files?.[0] ?? null);
input.click(); input.click();
}); });
@@ -601,7 +608,7 @@ function chooseWebFile(accept: string): Promise<File | null> {
} }
async function importOfflineZip() { async function importOfflineZip() {
if (importingZip.value) return; if (agentImportBusy.value) return;
let selected: string | File | null = null; let selected: string | File | null = null;
if (isWeb) { if (isWeb) {
selected = await chooseWebOfflineZip(); selected = await chooseWebOfflineZip();
@@ -610,7 +617,7 @@ async function importOfflineZip() {
const path = await open({ const path = await open({
title: t("driverStore.chooseOfflineDriverPackage"), title: t("driverStore.chooseOfflineDriverPackage"),
multiple: false, multiple: false,
filters: [{ name: "ZIP", extensions: ["zip"] }], filters: [{ name: "Driver package", extensions: ["zip", "zst"] }],
}); });
selected = typeof path === "string" ? path : null; selected = typeof path === "string" ? path : null;
} }
@@ -632,6 +639,7 @@ async function importOfflineZip() {
} }
async function importDriverFile(driver: AgentDriverInfo) { async function importDriverFile(driver: AgentDriverInfo) {
if (agentImportBusy.value) return;
const dbType = driver.db_type; const dbType = driver.db_type;
if (isPrestoSqlBuiltinDriver(dbType)) { if (isPrestoSqlBuiltinDriver(dbType)) {
await importJdbcDrivers(); await importJdbcDrivers();
@@ -663,30 +671,32 @@ async function importDriverFile(driver: AgentDriverInfo) {
toast(t("driverStore.driverImportSuccess", { label })); toast(t("driverStore.driverImportSuccess", { label }));
} }
}; };
const runSelectedFileImport = async (selected: string | File) => {
importingDriver.value = dbType;
try {
await installSelectedFile(selected);
} catch (e: any) {
toast(t("driverStore.driverImportFailed", { label, error: backendError(e) }));
} finally {
importingDriver.value = null;
}
};
if (isWeb) { if (isWeb) {
// Native release assets have no extension on macOS/Linux, so do not apply // Native release assets have no extension on macOS/Linux, so do not apply
// a browser accept filter that would make the correct file unselectable. // a browser accept filter that would make the correct file unselectable.
const file = await chooseWebFile(webDriverImportAccept(requiresJavaRuntime, isWindows)); const file = await chooseWebFile(webDriverImportAccept(requiresJavaRuntime, isWindows));
if (!file) return; if (!file) return;
try { await runSelectedFileImport(file);
await installSelectedFile(file);
} catch (e: any) {
toast(t("driverStore.driverImportFailed", { label, error: backendError(e) }));
}
return; return;
} }
const { open } = await import("@tauri-apps/plugin-dialog"); const { open } = await import("@tauri-apps/plugin-dialog");
const selected = await open({ const selected = await open({
title: t("driverStore.chooseDriverJar"), title: t("driverStore.chooseDriverJar"),
multiple: false, multiple: false,
filters: requiresJavaRuntime ? [{ name: "Driver package or JAR", extensions: ["zip", "jar"] }] : isWindows ? [{ name: "Driver package or executable", extensions: ["zip", "exe"] }] : undefined, filters: requiresJavaRuntime ? [{ name: "Driver package or JAR", extensions: ["zip", "zst", "jar"] }] : isWindows ? [{ name: "Driver package or executable", extensions: ["zip", "zst", "exe"] }] : undefined,
}); });
if (typeof selected !== "string") return; if (typeof selected !== "string") return;
try { await runSelectedFileImport(selected);
await installSelectedFile(selected);
} catch (e: any) {
toast(t("driverStore.driverImportFailed", { label, error: backendError(e) }));
}
} }
async function reinstallJre(jreKey: string) { async function reinstallJre(jreKey: string) {
@@ -1337,9 +1347,10 @@ watch(driverStoreTab, (tab) => {
</SelectContent> </SelectContent>
</Select> </Select>
</div> </div>
<Button variant="ghost" size="sm" class="h-7 rounded-md text-xs gap-1 text-muted-foreground" :disabled="importingZip || installing !== null || upgradingAll || reinstallingJre !== null || queuedDriverInstalls.length > 0" @click="importOfflineZip"> <Button variant="ghost" size="sm" class="h-7 rounded-md text-xs gap-1 text-muted-foreground" :disabled="agentImportBusy || installing !== null || upgradingAll || reinstallingJre !== null || queuedDriverInstalls.length > 0" @click="importOfflineZip">
<FileUp class="h-3.5 w-3.5" /> <Loader2 v-if="agentImportBusy" class="h-3.5 w-3.5 animate-spin" />
{{ importingZip ? t("driverStore.importing") : t("driverStore.importOfflinePackage") }} <FileUp v-else class="h-3.5 w-3.5" />
{{ agentImportBusy ? t("driverStore.importing") : t("driverStore.importOfflinePackage") }}
</Button> </Button>
<Button variant="ghost" size="sm" class="h-7 rounded-md text-xs gap-1 text-muted-foreground" :disabled="refreshing" @click="forceRefresh"> <Button variant="ghost" size="sm" class="h-7 rounded-md text-xs gap-1 text-muted-foreground" :disabled="refreshing" @click="forceRefresh">
<RefreshCw class="h-3.5 w-3.5" :class="{ 'animate-spin': refreshing }" /> <RefreshCw class="h-3.5 w-3.5" :class="{ 'animate-spin': refreshing }" />
@@ -1389,15 +1400,15 @@ watch(driverStoreTab, (tab) => {
<Check v-if="jre.installed" class="h-4 w-4 text-green-600" /> <Check v-if="jre.installed" class="h-4 w-4 text-green-600" />
<span v-else class="text-xs text-muted-foreground">{{ t("driverStore.notInstalled") }}</span> <span v-else class="text-xs text-muted-foreground">{{ t("driverStore.notInstalled") }}</span>
<DriverInstallProgressCircle v-if="reinstallingJre === jre.key" :percent="getJreReinstallPercent()" :title="getJreReinstallTitle(jre.installed ? t('driverStore.reinstalling') : t('driverStore.installing'))" /> <DriverInstallProgressCircle v-if="reinstallingJre === jre.key" :percent="getJreReinstallPercent()" :title="getJreReinstallTitle(jre.installed ? t('driverStore.reinstalling') : t('driverStore.installing'))" />
<Button v-else-if="!jre.installed" type="button" variant="default" size="sm" class="h-8 rounded-md text-xs" :disabled="reinstallingJre !== null || installing !== null || importingZip" @click="reinstallJre(jre.key)"> <Button v-else-if="!jre.installed" type="button" variant="default" size="sm" class="h-8 rounded-md text-xs" :disabled="reinstallingJre !== null || installing !== null || agentImportBusy" @click="reinstallJre(jre.key)">
<Download class="h-3.5 w-3.5 mr-1" /> <Download class="h-3.5 w-3.5 mr-1" />
{{ t("driverStore.install") }} {{ t("driverStore.install") }}
</Button> </Button>
<Button v-else-if="jre.installed" type="button" variant="outline" size="sm" class="h-8 rounded-md text-xs" :disabled="reinstallingJre !== null || installing !== null || importingZip" @click="reinstallJre(jre.key)"> <Button v-else-if="jre.installed" type="button" variant="outline" size="sm" class="h-8 rounded-md text-xs" :disabled="reinstallingJre !== null || installing !== null || agentImportBusy" @click="reinstallJre(jre.key)">
<RotateCcw class="h-3.5 w-3.5 mr-1" /> <RotateCcw class="h-3.5 w-3.5 mr-1" />
{{ t("driverStore.reinstall") }} {{ t("driverStore.reinstall") }}
</Button> </Button>
<Button v-if="jre.installed" type="button" variant="ghost" size="sm" class="h-8 rounded-md text-xs text-muted-foreground hover:text-destructive" :disabled="reinstallingJre !== null || installing !== null || importingZip" @click="uninstallJre(jre.key)"> <Button v-if="jre.installed" type="button" variant="ghost" size="sm" class="h-8 rounded-md text-xs text-muted-foreground hover:text-destructive" :disabled="reinstallingJre !== null || installing !== null || agentImportBusy" @click="uninstallJre(jre.key)">
{{ t("driverStore.uninstall") }} {{ t("driverStore.uninstall") }}
</Button> </Button>
</div> </div>
@@ -1417,7 +1428,7 @@ watch(driverStoreTab, (tab) => {
<div class="text-sm font-semibold">{{ t("driverStore.updatesAvailableTitle") }} ({{ globalUpdatableDrivers.length }})</div> <div class="text-sm font-semibold">{{ t("driverStore.updatesAvailableTitle") }} ({{ globalUpdatableDrivers.length }})</div>
<p class="text-xs text-muted-foreground">{{ t("driverStore.updatesAvailableDescription") }}</p> <p class="text-xs text-muted-foreground">{{ t("driverStore.updatesAvailableDescription") }}</p>
</div> </div>
<Button size="sm" class="h-7 rounded-md text-xs shrink-0 ml-3" :disabled="installing !== null || upgradingAll || importingZip" @click="upgradeAll"> <Button size="sm" class="h-7 rounded-md text-xs shrink-0 ml-3" :disabled="installing !== null || upgradingAll || agentImportBusy" @click="upgradeAll">
<Loader2 v-if="upgradingAll" class="h-3 w-3 animate-spin mr-1" /> <Loader2 v-if="upgradingAll" class="h-3 w-3 animate-spin mr-1" />
<Download v-else class="h-3 w-3 mr-1" /> <Download v-else class="h-3 w-3 mr-1" />
{{ upgradingAll ? t("driverStore.upgradingProgress", { current: upgradingCompletedCount, total: upgradingTotal }) : t("driverStore.upgradeAll") }} {{ upgradingAll ? t("driverStore.upgradingProgress", { current: upgradingCompletedCount, total: upgradingTotal }) : t("driverStore.upgradeAll") }}
@@ -1449,14 +1460,14 @@ watch(driverStoreTab, (tab) => {
size="sm" size="sm"
variant="outline" variant="outline"
class="h-7 rounded-md border-green-500/30 bg-green-500/10 text-xs text-green-700 hover:bg-green-500/15" class="h-7 rounded-md border-green-500/30 bg-green-500/10 text-xs text-green-700 hover:bg-green-500/15"
:disabled="upgradingAll || importingZip" :disabled="upgradingAll || agentImportBusy"
@click="removeQueuedDriverInstall(driver.db_type)" @click="removeQueuedDriverInstall(driver.db_type)"
> >
<Clock3 class="h-3 w-3 mr-1" /> <Clock3 class="h-3 w-3 mr-1" />
{{ t("driverStore.queued") }} {{ t("driverStore.queued") }}
</Button> </Button>
<DriverInstallProgressCircle v-else-if="!driver.installed && isDriverProgressActive(driver.db_type)" :percent="getAgentProgressPercent(driver.db_type)" :title="getAgentProgressTitle(driver.db_type, t('driverStore.installing'))" /> <DriverInstallProgressCircle v-else-if="!driver.installed && isDriverProgressActive(driver.db_type)" :percent="getAgentProgressPercent(driver.db_type)" :title="getAgentProgressTitle(driver.db_type, t('driverStore.installing'))" />
<Button v-else-if="!driver.installed" size="sm" class="h-7 rounded-md text-xs" :disabled="upgradingAll || importingZip" @click="installDriver(driver.db_type)"> <Button v-else-if="!driver.installed" size="sm" class="h-7 rounded-md text-xs" :disabled="upgradingAll || agentImportBusy" @click="installDriver(driver.db_type)">
<Download class="h-3 w-3 mr-1" /> <Download class="h-3 w-3 mr-1" />
{{ t("driverStore.install") }} {{ t("driverStore.install") }}
</Button> </Button>
@@ -1465,11 +1476,12 @@ watch(driverStoreTab, (tab) => {
size="sm" size="sm"
variant="ghost" variant="ghost"
class="h-7 w-7 rounded-md text-xs text-muted-foreground" class="h-7 w-7 rounded-md text-xs text-muted-foreground"
:title="t('driverStore.importLocalJar')" :title="importingDriver === driver.db_type ? t('driverStore.importing') : t('driverStore.importLocalJar')"
:disabled="upgradingAll || installing !== null || importingZip" :disabled="upgradingAll || installing !== null || agentImportBusy"
@click="importDriverFile(driver)" @click="importDriverFile(driver)"
> >
<FileUp class="h-3.5 w-3.5" /> <Loader2 v-if="importingDriver === driver.db_type" class="h-3.5 w-3.5 animate-spin" />
<FileUp v-else class="h-3.5 w-3.5" />
</Button> </Button>
<Check v-if="driver.installed && !(driver.update_available && isDriverProgressActive(driver.db_type))" class="h-4 w-4 text-green-600" /> <Check v-if="driver.installed && !(driver.update_available && isDriverProgressActive(driver.db_type))" class="h-4 w-4 text-green-600" />
<Button <Button
@@ -1477,17 +1489,17 @@ watch(driverStoreTab, (tab) => {
size="sm" size="sm"
variant="outline" variant="outline"
class="h-7 rounded-md border-green-500/30 bg-green-500/10 text-xs text-green-700 hover:bg-green-500/15" class="h-7 rounded-md border-green-500/30 bg-green-500/10 text-xs text-green-700 hover:bg-green-500/15"
:disabled="upgradingAll || importingZip" :disabled="upgradingAll || agentImportBusy"
@click="removeQueuedDriverInstall(driver.db_type)" @click="removeQueuedDriverInstall(driver.db_type)"
> >
<Clock3 class="h-3 w-3 mr-1" /> <Clock3 class="h-3 w-3 mr-1" />
{{ t("driverStore.queued") }} {{ t("driverStore.queued") }}
</Button> </Button>
<DriverInstallProgressCircle v-else-if="driver.installed && driver.update_available && isDriverProgressActive(driver.db_type)" :percent="getAgentProgressPercent(driver.db_type)" :title="getAgentProgressTitle(driver.db_type, t('driverStore.updating'))" /> <DriverInstallProgressCircle v-else-if="driver.installed && driver.update_available && isDriverProgressActive(driver.db_type)" :percent="getAgentProgressPercent(driver.db_type)" :title="getAgentProgressTitle(driver.db_type, t('driverStore.updating'))" />
<Button v-else-if="driver.installed && driver.update_available" size="sm" variant="outline" class="h-7 rounded-md text-xs" :disabled="upgradingAll || importingZip" @click="installDriver(driver.db_type)"> <Button v-else-if="driver.installed && driver.update_available" size="sm" variant="outline" class="h-7 rounded-md text-xs" :disabled="upgradingAll || agentImportBusy" @click="installDriver(driver.db_type)">
{{ t("driverStore.update") }} {{ t("driverStore.update") }}
</Button> </Button>
<Button v-if="driver.installed" variant="ghost" size="sm" class="h-7 rounded-md text-xs text-muted-foreground hover:text-destructive" :disabled="installing !== null || upgradingAll || importingZip || isDriverQueued(driver.db_type)" @click="uninstallDriver(driver.db_type)"> <Button v-if="driver.installed" variant="ghost" size="sm" class="h-7 rounded-md text-xs text-muted-foreground hover:text-destructive" :disabled="installing !== null || upgradingAll || agentImportBusy || isDriverQueued(driver.db_type)" @click="uninstallDriver(driver.db_type)">
{{ t("driverStore.uninstall") }} {{ t("driverStore.uninstall") }}
</Button> </Button>
</div> </div>
@@ -1566,14 +1578,14 @@ watch(driverStoreTab, (tab) => {
size="sm" size="sm"
variant="outline" variant="outline"
class="h-7 rounded-md border-green-500/30 bg-green-500/10 text-xs text-green-700 hover:bg-green-500/15" class="h-7 rounded-md border-green-500/30 bg-green-500/10 text-xs text-green-700 hover:bg-green-500/15"
:disabled="upgradingAll || importingZip" :disabled="upgradingAll || agentImportBusy"
@click="removeQueuedDriverInstall(driver.db_type)" @click="removeQueuedDriverInstall(driver.db_type)"
> >
<Clock3 class="h-3 w-3 mr-1" /> <Clock3 class="h-3 w-3 mr-1" />
{{ t("driverStore.queued") }} {{ t("driverStore.queued") }}
</Button> </Button>
<DriverInstallProgressCircle v-else-if="!driver.installed && isDriverProgressActive(driver.db_type)" :percent="getAgentProgressPercent(driver.db_type)" :title="getAgentProgressTitle(driver.db_type, t('driverStore.installing'))" /> <DriverInstallProgressCircle v-else-if="!driver.installed && isDriverProgressActive(driver.db_type)" :percent="getAgentProgressPercent(driver.db_type)" :title="getAgentProgressTitle(driver.db_type, t('driverStore.installing'))" />
<Button v-else-if="!driver.installed" size="sm" class="h-7 rounded-md text-xs" :disabled="upgradingAll || importingZip" @click="installDriver(driver.db_type)"> <Button v-else-if="!driver.installed" size="sm" class="h-7 rounded-md text-xs" :disabled="upgradingAll || agentImportBusy" @click="installDriver(driver.db_type)">
<Download class="h-3 w-3 mr-1" /> <Download class="h-3 w-3 mr-1" />
{{ t("driverStore.install") }} {{ t("driverStore.install") }}
</Button> </Button>
@@ -1582,11 +1594,12 @@ watch(driverStoreTab, (tab) => {
size="sm" size="sm"
variant="ghost" variant="ghost"
class="h-7 w-7 rounded-md text-xs text-muted-foreground" class="h-7 w-7 rounded-md text-xs text-muted-foreground"
:title="t('driverStore.importLocalJar')" :title="importingDriver === driver.db_type ? t('driverStore.importing') : t('driverStore.importLocalJar')"
:disabled="upgradingAll || installing !== null || importingZip" :disabled="upgradingAll || installing !== null || agentImportBusy"
@click="importDriverFile(driver)" @click="importDriverFile(driver)"
> >
<FileUp class="h-3.5 w-3.5" /> <Loader2 v-if="importingDriver === driver.db_type" class="h-3.5 w-3.5 animate-spin" />
<FileUp v-else class="h-3.5 w-3.5" />
</Button> </Button>
<Check v-if="driver.installed && !(driver.update_available && isDriverProgressActive(driver.db_type))" class="h-4 w-4 text-green-600" /> <Check v-if="driver.installed && !(driver.update_available && isDriverProgressActive(driver.db_type))" class="h-4 w-4 text-green-600" />
<Button <Button
@@ -1594,17 +1607,24 @@ watch(driverStoreTab, (tab) => {
size="sm" size="sm"
variant="outline" variant="outline"
class="h-7 rounded-md border-green-500/30 bg-green-500/10 text-xs text-green-700 hover:bg-green-500/15" class="h-7 rounded-md border-green-500/30 bg-green-500/10 text-xs text-green-700 hover:bg-green-500/15"
:disabled="upgradingAll || importingZip" :disabled="upgradingAll || agentImportBusy"
@click="removeQueuedDriverInstall(driver.db_type)" @click="removeQueuedDriverInstall(driver.db_type)"
> >
<Clock3 class="h-3 w-3 mr-1" /> <Clock3 class="h-3 w-3 mr-1" />
{{ t("driverStore.queued") }} {{ t("driverStore.queued") }}
</Button> </Button>
<DriverInstallProgressCircle v-else-if="driver.installed && driver.update_available && isDriverProgressActive(driver.db_type)" :percent="getAgentProgressPercent(driver.db_type)" :title="getAgentProgressTitle(driver.db_type, t('driverStore.updating'))" /> <DriverInstallProgressCircle v-else-if="driver.installed && driver.update_available && isDriverProgressActive(driver.db_type)" :percent="getAgentProgressPercent(driver.db_type)" :title="getAgentProgressTitle(driver.db_type, t('driverStore.updating'))" />
<Button v-else-if="driver.installed && driver.update_available" size="sm" variant="outline" class="h-7 rounded-md text-xs" :disabled="upgradingAll || importingZip" @click="installDriver(driver.db_type)"> <Button v-else-if="driver.installed && driver.update_available" size="sm" variant="outline" class="h-7 rounded-md text-xs" :disabled="upgradingAll || agentImportBusy" @click="installDriver(driver.db_type)">
{{ t("driverStore.update") }} {{ t("driverStore.update") }}
</Button> </Button>
<Button v-if="driver.installed" variant="ghost" size="sm" class="h-7 rounded-md text-xs text-muted-foreground hover:text-destructive" :disabled="installing !== null || upgradingAll || importingZip || isDriverQueued(driver.db_type)" @click="uninstallDriver(driver.db_type)"> <Button
v-if="driver.installed"
variant="ghost"
size="sm"
class="h-7 rounded-md text-xs text-muted-foreground hover:text-destructive"
:disabled="installing !== null || upgradingAll || agentImportBusy || isDriverQueued(driver.db_type)"
@click="uninstallDriver(driver.db_type)"
>
{{ t("driverStore.uninstall") }} {{ t("driverStore.uninstall") }}
</Button> </Button>
</div> </div>
@@ -1642,14 +1662,14 @@ watch(driverStoreTab, (tab) => {
size="sm" size="sm"
variant="outline" variant="outline"
class="h-7 rounded-md border-green-500/30 bg-green-500/10 text-xs text-green-700 hover:bg-green-500/15" class="h-7 rounded-md border-green-500/30 bg-green-500/10 text-xs text-green-700 hover:bg-green-500/15"
:disabled="upgradingAll || importingZip" :disabled="upgradingAll || agentImportBusy"
@click="removeQueuedDriverInstall(driver.db_type)" @click="removeQueuedDriverInstall(driver.db_type)"
> >
<Clock3 class="h-3 w-3 mr-1" /> <Clock3 class="h-3 w-3 mr-1" />
{{ t("driverStore.queued") }} {{ t("driverStore.queued") }}
</Button> </Button>
<DriverInstallProgressCircle v-else-if="!driver.installed && isDriverProgressActive(driver.db_type)" :percent="getAgentProgressPercent(driver.db_type)" :title="getAgentProgressTitle(driver.db_type, t('driverStore.installing'))" /> <DriverInstallProgressCircle v-else-if="!driver.installed && isDriverProgressActive(driver.db_type)" :percent="getAgentProgressPercent(driver.db_type)" :title="getAgentProgressTitle(driver.db_type, t('driverStore.installing'))" />
<Button v-else-if="!driver.installed" size="sm" class="h-7 rounded-md text-xs" :disabled="upgradingAll || importingZip" @click="installDriver(driver.db_type)"> <Button v-else-if="!driver.installed" size="sm" class="h-7 rounded-md text-xs" :disabled="upgradingAll || agentImportBusy" @click="installDriver(driver.db_type)">
<Download class="h-3 w-3 mr-1" /> <Download class="h-3 w-3 mr-1" />
{{ t("driverStore.install") }} {{ t("driverStore.install") }}
</Button> </Button>
@@ -1658,11 +1678,12 @@ watch(driverStoreTab, (tab) => {
size="sm" size="sm"
variant="ghost" variant="ghost"
class="h-7 w-7 rounded-md text-xs text-muted-foreground" class="h-7 w-7 rounded-md text-xs text-muted-foreground"
:title="t('driverStore.importLocalJar')" :title="importingDriver === driver.db_type ? t('driverStore.importing') : t('driverStore.importLocalJar')"
:disabled="upgradingAll || installing !== null || importingZip" :disabled="upgradingAll || installing !== null || agentImportBusy"
@click="importDriverFile(driver)" @click="importDriverFile(driver)"
> >
<FileUp class="h-3.5 w-3.5" /> <Loader2 v-if="importingDriver === driver.db_type" class="h-3.5 w-3.5 animate-spin" />
<FileUp v-else class="h-3.5 w-3.5" />
</Button> </Button>
<Check v-if="driver.installed && !(driver.update_available && isDriverProgressActive(driver.db_type))" class="h-4 w-4 text-green-600" /> <Check v-if="driver.installed && !(driver.update_available && isDriverProgressActive(driver.db_type))" class="h-4 w-4 text-green-600" />
<Button <Button
@@ -1670,17 +1691,17 @@ watch(driverStoreTab, (tab) => {
size="sm" size="sm"
variant="outline" variant="outline"
class="h-7 rounded-md border-green-500/30 bg-green-500/10 text-xs text-green-700 hover:bg-green-500/15" class="h-7 rounded-md border-green-500/30 bg-green-500/10 text-xs text-green-700 hover:bg-green-500/15"
:disabled="upgradingAll || importingZip" :disabled="upgradingAll || agentImportBusy"
@click="removeQueuedDriverInstall(driver.db_type)" @click="removeQueuedDriverInstall(driver.db_type)"
> >
<Clock3 class="h-3 w-3 mr-1" /> <Clock3 class="h-3 w-3 mr-1" />
{{ t("driverStore.queued") }} {{ t("driverStore.queued") }}
</Button> </Button>
<DriverInstallProgressCircle v-else-if="driver.installed && driver.update_available && isDriverProgressActive(driver.db_type)" :percent="getAgentProgressPercent(driver.db_type)" :title="getAgentProgressTitle(driver.db_type, t('driverStore.updating'))" /> <DriverInstallProgressCircle v-else-if="driver.installed && driver.update_available && isDriverProgressActive(driver.db_type)" :percent="getAgentProgressPercent(driver.db_type)" :title="getAgentProgressTitle(driver.db_type, t('driverStore.updating'))" />
<Button v-else-if="driver.installed && driver.update_available" size="sm" variant="outline" class="h-7 rounded-md text-xs" :disabled="upgradingAll || importingZip" @click="installDriver(driver.db_type)"> <Button v-else-if="driver.installed && driver.update_available" size="sm" variant="outline" class="h-7 rounded-md text-xs" :disabled="upgradingAll || agentImportBusy" @click="installDriver(driver.db_type)">
{{ t("driverStore.update") }} {{ t("driverStore.update") }}
</Button> </Button>
<Button v-if="driver.installed" variant="ghost" size="sm" class="h-7 rounded-md text-xs text-muted-foreground hover:text-destructive" :disabled="installing !== null || upgradingAll || importingZip || isDriverQueued(driver.db_type)" @click="uninstallDriver(driver.db_type)"> <Button v-if="driver.installed" variant="ghost" size="sm" class="h-7 rounded-md text-xs text-muted-foreground hover:text-destructive" :disabled="installing !== null || upgradingAll || agentImportBusy || isDriverQueued(driver.db_type)" @click="uninstallDriver(driver.db_type)">
{{ t("driverStore.uninstall") }} {{ t("driverStore.uninstall") }}
</Button> </Button>
</div> </div>
+3 -3
View File
@@ -483,8 +483,8 @@ export async function upgradeAllAgents(_source?: UpdateDownloadSource, operation
return post("/api/agents/upgrade-all", { operationId }); return post("/api/agents/upgrade-all", { operationId });
} }
export async function checkAgentUpdateBlockers(_dbTypes: string[]): Promise<AgentUpdateBlocker[]> { export async function checkAgentUpdateBlockers(dbTypes: string[]): Promise<AgentUpdateBlocker[]> {
return []; return post("/api/agents/update-blockers", { dbTypes });
} }
export async function uninstallAgent(dbType: string): Promise<void> { export async function uninstallAgent(dbType: string): Promise<void> {
@@ -505,7 +505,7 @@ export async function invalidateAgentRegistryCache(): Promise<void> {
export async function importAgentsFromZip(fileOrPath: string | File, operationId?: string): Promise<number> { export async function importAgentsFromZip(fileOrPath: string | File, operationId?: string): Promise<number> {
if (typeof fileOrPath === "string") { if (typeof fileOrPath === "string") {
throw new Error("Offline ZIP import in web mode requires a File object, not a file path"); throw new Error("Offline package import in web mode requires a File object, not a file path");
} }
const formData = new FormData(); const formData = new FormData();
if (operationId) formData.append("operationId", operationId); if (operationId) formData.append("operationId", operationId);
+1 -1
View File
@@ -1740,7 +1740,7 @@ export async function invalidateAgentRegistryCache(): Promise<void> {
export async function importAgentsFromZip(path: string | File, operationId?: string): Promise<number> { export async function importAgentsFromZip(path: string | File, operationId?: string): Promise<number> {
if (typeof path !== "string") { if (typeof path !== "string") {
throw new Error("Desktop offline ZIP import requires a local file path"); throw new Error("Desktop offline package import requires a local file path");
} }
return invoke("import_agents_from_zip", { path, operationId }); return invoke("import_agents_from_zip", { path, operationId });
} }
@@ -1,5 +1,13 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import { driverStoreFocusForInstallError, hasInstalledAgentVersion } from "@/lib/connection/agentDriverInstallHint"; import { agentDriverInstallKey, driverStoreFocusForInstallError, hasInstalledAgentVersion, showAgentDriverInstallHint } from "@/lib/connection/agentDriverInstallHint";
describe("DuckDB driver installation", () => {
it("requires the downloadable DuckDB driver before connecting", () => {
expect(agentDriverInstallKey("duckdb")).toBe("duckdb");
expect(showAgentDriverInstallHint("duckdb", [])).toBe(true);
expect(showAgentDriverInstallHint("duckdb", [{ db_type: "duckdb", installed: true }])).toBe(false);
});
});
describe("driverStoreFocusForInstallError", () => { describe("driverStoreFocusForInstallError", () => {
it("focuses the missing agent driver for driver-not-installed errors", () => { it("focuses the missing agent driver for driver-not-installed errors", () => {
@@ -10,7 +10,7 @@ describe("getCategoryForAgentDriver", () => {
// domestic // domestic
expect(getCategoryForAgentDriver("dameng")).toBe("domestic"); expect(getCategoryForAgentDriver("dameng")).toBe("domestic");
// lightweight // lightweight
expect(getCategoryForAgentDriver("access")).toBe("lightweight"); expect(getCategoryForAgentDriver("duckdb")).toBe("lightweight");
// document // document
expect(getCategoryForAgentDriver("mongodb")).toBe("document"); expect(getCategoryForAgentDriver("mongodb")).toBe("document");
// graph_ai // graph_ai
@@ -69,8 +69,8 @@ describe("AGENT_DRIVER_CATEGORY_MAP integrity", () => {
it("covers all known agent driver keys with no stale entries", () => { it("covers all known agent driver keys with no stale entries", () => {
// Golden set — mirrors the store-visible entries in agent_catalog.rs // Golden set — mirrors the store-visible entries in agent_catalog.rs
// plus EXTRA_DRIVER_STORE_ENTRIES (kafka, rocketmq, rabbitmq, // plus EXTRA_DRIVER_STORE_ENTRIES (duckdb, kafka, rocketmq,
// sqlserver-legacy) and the PrestoSQL built-in driver. // rabbitmq, sqlserver-legacy) and the PrestoSQL built-in driver.
// When an Agent driver is added to the catalog its key MUST appear here; // When an Agent driver is added to the catalog its key MUST appear here;
// removing a key from the map without removing it from the list below // removing a key from the map without removing it from the list below
// will fail this test. // will fail this test.
@@ -112,6 +112,7 @@ describe("AGENT_DRIVER_CATEGORY_MAP integrity", () => {
"yashandb", "yashandb",
// lightweight // lightweight
"access", "access",
"duckdb",
"h2", "h2",
"h2-legacy", "h2-legacy",
// document // document
@@ -24,6 +24,7 @@ export const AGENT_DRIVER_CATEGORY_MAP: Readonly<Record<string, DriverCategoryKe
databend: "analytics", databend: "analytics",
databricks: "analytics", databricks: "analytics",
db2: "sql", db2: "sql",
duckdb: "lightweight",
etcd: "registry_config", etcd: "registry_config",
exasol: "analytics", exasol: "analytics",
firebird: "sql", firebird: "sql",
@@ -2,15 +2,16 @@ import { describe, expect, it } from "vitest";
import { isOfflineDriverPackage, webDriverImportAccept } from "./driverImportSelection"; import { isOfflineDriverPackage, webDriverImportAccept } from "./driverImportSelection";
describe("driver import selection", () => { describe("driver import selection", () => {
it("recognizes ZIP paths and uploaded files case-insensitively", () => { it("recognizes ZIP and tar.zst packages case-insensitively", () => {
expect(isOfflineDriverPackage("C:\\Downloads\\dbx-agent-h2-0.2.5.ZIP")).toBe(true); expect(isOfflineDriverPackage("C:\\Downloads\\dbx-agent-h2-0.2.5.ZIP")).toBe(true);
expect(isOfflineDriverPackage({ name: "dbx-agent-kingbase-0.1.34-macos-aarch64.zip" })).toBe(true); expect(isOfflineDriverPackage({ name: "dbx-agent-kingbase-0.1.34-macos-aarch64.zip" })).toBe(true);
expect(isOfflineDriverPackage({ name: "dbx-agent-duckdb-0.1.0-macos-aarch64.TAR.ZST" })).toBe(true);
expect(isOfflineDriverPackage({ name: "dbx-agent-h2-0.2.5.jar" })).toBe(false); expect(isOfflineDriverPackage({ name: "dbx-agent-h2-0.2.5.jar" })).toBe(false);
}); });
it("allows ZIP alongside the platform raw artifact", () => { it("allows ZIP alongside the platform raw artifact", () => {
expect(webDriverImportAccept(true, false)).toBe(".zip,.jar"); expect(webDriverImportAccept(true, false)).toBe(".zip,.tar.zst,.jar");
expect(webDriverImportAccept(false, true)).toBe(".zip,.exe"); expect(webDriverImportAccept(false, true)).toBe(".zip,.tar.zst,.exe");
expect(webDriverImportAccept(false, false)).toBe(""); expect(webDriverImportAccept(false, false)).toBe("");
}); });
}); });
@@ -2,10 +2,11 @@ export type DriverImportSelection = string | { name: string };
export function isOfflineDriverPackage(selection: DriverImportSelection): boolean { export function isOfflineDriverPackage(selection: DriverImportSelection): boolean {
const name = typeof selection === "string" ? selection : selection.name; const name = typeof selection === "string" ? selection : selection.name;
return name.toLowerCase().endsWith(".zip"); const lowerName = name.toLowerCase();
return lowerName.endsWith(".zip") || lowerName.endsWith(".tar.zst");
} }
export function webDriverImportAccept(requiresJavaRuntime: boolean, isWindows: boolean): string { export function webDriverImportAccept(requiresJavaRuntime: boolean, isWindows: boolean): string {
if (requiresJavaRuntime) return ".zip,.jar"; if (requiresJavaRuntime) return ".zip,.tar.zst,.jar";
return isWindows ? ".zip,.exe" : ""; return isWindows ? ".zip,.tar.zst,.exe" : "";
} }
+3 -30
View File
@@ -4,36 +4,9 @@ version = "0.1.0"
edition = "2021" edition = "2021"
license = "Apache-2.0" license = "Apache-2.0"
[[bin]]
name = "duckdb-worker-test-host"
path = "tests/support/duckdb_worker_test_host.rs"
required-features = ["duckdb-bundled"]
test = false
bench = false
[[bin]]
name = "duckdb-worker-hanging-connect-test-host"
path = "tests/support/duckdb_worker_hanging_connect_test_host.rs"
required-features = ["duckdb-bundled"]
test = false
[[bin]]
name = "duckdb-worker-pid-test-host"
path = "tests/support/duckdb_worker_pid_test_host.rs"
required-features = ["duckdb-bundled"]
test = false
bench = false
[[bin]]
name = "duckdb-worker-file-lock-owner"
path = "tests/support/duckdb_worker_file_lock_owner.rs"
required-features = ["duckdb-bundled"]
test = false
bench = false
[features] [features]
default = ["duckdb-bundled", "mq-admin", "sqlite-sqlcipher", "system-fonts"] default = ["duckdb-sidecar", "mq-admin", "sqlite-sqlcipher", "system-fonts"]
duckdb-bundled = ["duckdb/bundled", "duckdb/parquet"] duckdb-sidecar = []
mq-admin = [] mq-admin = []
sqlite-sqlcipher = ["rusqlite/bundled-sqlcipher-vendored-openssl"] sqlite-sqlcipher = ["rusqlite/bundled-sqlcipher-vendored-openssl"]
openapi = ["dep:utoipa"] openapi = ["dep:utoipa"]
@@ -71,7 +44,6 @@ sqlparser = { version = "0.62.0", features = ["visitor"] }
redis = { version = "0.32.2", features = ["tokio-comp", "tls-rustls", "tls-rustls-insecure", "tokio-rustls-comp", "sentinel", "cluster-async"] } redis = { version = "0.32.2", features = ["tokio-comp", "tls-rustls", "tls-rustls-insecure", "tokio-rustls-comp", "sentinel", "cluster-async"] }
rustls = { version = "0.23", features = ["aws-lc-rs"] } rustls = { version = "0.23", features = ["aws-lc-rs"] }
rustls-pemfile = "2.2" rustls-pemfile = "2.2"
duckdb = { version = "1.3.2", optional = true }
tiberius = { version = "0.12.3", default-features = false, features = ["tds73", "chrono", "rust_decimal", "rustls", "sql-browser-tokio"] } tiberius = { version = "0.12.3", default-features = false, features = ["tds73", "chrono", "rust_decimal", "rustls", "sql-browser-tokio"] }
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls", "socks"] } reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls", "socks"] }
prometheus-parse = "=0.2.5" prometheus-parse = "=0.2.5"
@@ -98,6 +70,7 @@ font-kit = { version = "0.14.3", optional = true }
zip = { version = "4", default-features = false, features = ["deflate"] } zip = { version = "4", default-features = false, features = ["deflate"] }
flate2 = "1" flate2 = "1"
tar = "0.4" tar = "0.4"
zstd = "0.13"
sysinfo = { version = "0.32", features = ["system"] } sysinfo = { version = "0.32", features = ["system"] }
serde_yaml = "0.9" serde_yaml = "0.9"
minijinja = "2" minijinja = "2"
@@ -199,7 +199,7 @@
"fieldLineage": false, "fieldLineage": false,
"sqlExplain": false, "sqlExplain": false,
"userAdmin": false, "userAdmin": false,
"driverManagement": false "driverManagement": true
} }
}, },
{ {
+9
View File
@@ -40,12 +40,14 @@ const H2_PROFILES: &[AgentDriverProfile] =
&[AgentDriverProfile { profile: "h2-legacy", key: "h2-legacy", label: "H2 2.1 Legacy", store_visible: true }]; &[AgentDriverProfile { profile: "h2-legacy", key: "h2-legacy", label: "H2 2.1 Legacy", store_visible: true }];
const EXTRA_AGENT_LABELS: &[(&str, &str)] = &[ const EXTRA_AGENT_LABELS: &[(&str, &str)] = &[
("duckdb", "DuckDB"),
("kafka", "Apache Kafka"), ("kafka", "Apache Kafka"),
("rocketmq", "Apache RocketMQ"), ("rocketmq", "Apache RocketMQ"),
("rabbitmq", "RabbitMQ"), ("rabbitmq", "RabbitMQ"),
("sqlserver-legacy", "SQL Server legacy compatibility component"), ("sqlserver-legacy", "SQL Server legacy compatibility component"),
]; ];
const EXTRA_DRIVER_STORE_ENTRIES: &[(&str, &str)] = &[ const EXTRA_DRIVER_STORE_ENTRIES: &[(&str, &str)] = &[
("duckdb", "DuckDB"),
("kafka", "Apache Kafka"), ("kafka", "Apache Kafka"),
("rocketmq", "Apache RocketMQ"), ("rocketmq", "Apache RocketMQ"),
("rabbitmq", "RabbitMQ"), ("rabbitmq", "RabbitMQ"),
@@ -388,4 +390,11 @@ mod tests {
assert_eq!(agent_key(&DatabaseType::H2, Some("h2-legacy")), Some("h2-legacy")); assert_eq!(agent_key(&DatabaseType::H2, Some("h2-legacy")), Some("h2-legacy"));
assert!(driver_store_entries().any(|(key, label)| key == "h2-legacy" && label == "H2 2.1 Legacy")); assert!(driver_store_entries().any(|(key, label)| key == "h2-legacy" && label == "H2 2.1 Legacy"));
} }
#[test]
fn duckdb_is_available_in_driver_store_without_using_agent_runtime() {
assert!(driver_store_entries().any(|(key, label)| key == "duckdb" && label == "DuckDB"));
assert_eq!(label_for_key("duckdb"), Some("DuckDB"));
assert!(!is_agent_type(&DatabaseType::DuckDb));
}
} }
+8
View File
@@ -345,6 +345,14 @@ pub struct DriverInfo {
pub struct ArtifactInfo { pub struct ArtifactInfo {
pub url: String, pub url: String,
pub size: u64, pub size: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub format: Option<ArtifactFormat>,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum ArtifactFormat {
TarZstd,
} }
#[derive(Debug, Clone, Serialize, Deserialize, Default)] #[derive(Debug, Clone, Serialize, Deserialize, Default)]
+670 -30
View File
@@ -8,7 +8,7 @@ use futures::{stream, StreamExt};
use crate::agent_catalog; use crate::agent_catalog;
use crate::agent_manager::{ use crate::agent_manager::{
AgentDriverInfo, AgentManager, AgentRegistry, InstalledDriver, JavaRuntimeMode, DEFAULT_JRE_KEY, AgentDriverInfo, AgentManager, AgentRegistry, ArtifactFormat, InstalledDriver, JavaRuntimeMode, DEFAULT_JRE_KEY,
}; };
use crate::DownloadSource; use crate::DownloadSource;
@@ -577,7 +577,7 @@ pub async fn reinstall_agent_jre_from(
.platforms .platforms
.get(platform) .get(platform)
.ok_or_else(|| format!("No JRE {jre_key} available for platform: {platform}"))?; .ok_or_else(|| format!("No JRE {jre_key} available for platform: {platform}"))?;
let jre_archive = am.base_dir().join("jre-download.tar.gz"); let jre_archive = jre_archive_download_path(am, jre_key, platform_jre.format);
download_with_progress( download_with_progress(
am, am,
&progress, &progress,
@@ -600,7 +600,7 @@ pub async fn reinstall_agent_jre_from(
am.stop_daemons().await; am.stop_daemons().await;
let stash = replace_old_jre_dir(&jre_dir)?; let stash = replace_old_jre_dir(&jre_dir)?;
persist_pending_jre_cleanup(am, stash.as_ref()).await?; persist_pending_jre_cleanup(am, stash.as_ref()).await?;
extract_tar_gz(&jre_archive, &jre_dir)?; extract_jre_archive(&jre_archive, &jre_dir, platform_jre.format)?;
std::fs::remove_file(&jre_archive).ok(); std::fs::remove_file(&jre_archive).ok();
am.mutate_state(|state| state.jre_versions.insert(jre_key.to_string(), jre_info.version.clone()))?; am.mutate_state(|state| state.jre_versions.insert(jre_key.to_string(), jre_info.version.clone()))?;
cleanup_jre_download_cache_after_success(am, jre_key); cleanup_jre_download_cache_after_success(am, jre_key);
@@ -627,6 +627,41 @@ pub async fn import_agents_from_zip(
.await .await
} }
pub fn inspect_offline_package(package_path: &Path) -> Result<OfflineImportPlan, String> {
if is_tar_zstd_package(package_path) {
inspect_tar_zstd_driver_package(package_path)
} else {
inspect_offline_zip(package_path)
}
}
pub async fn import_agents_from_package(
am: &AgentManager,
package_path: &Path,
progress: impl Fn(AgentProgressEvent),
) -> Result<OfflineImportResult, String> {
if is_tar_zstd_package(package_path) {
import_tar_zstd_driver_package(am, package_path, |event| {
progress(AgentProgressEvent {
operation_id: None,
step: event.step,
downloaded: Some(event.current as u64),
total: Some(event.total as u64),
db_type: event.db_type,
current: Some(event.current),
total_drivers: Some(event.total),
});
})
.await
} else {
import_agents_from_zip(am, package_path, progress).await
}
}
fn is_tar_zstd_package(path: &Path) -> bool {
path.file_name().and_then(|name| name.to_str()).is_some_and(|name| name.to_ascii_lowercase().ends_with(".tar.zst"))
}
async fn install_agent_driver_with_batch( async fn install_agent_driver_with_batch(
am: &AgentManager, am: &AgentManager,
db_type: &str, db_type: &str,
@@ -734,7 +769,7 @@ async fn ensure_jre_from_registry(
.platforms .platforms
.get(platform) .get(platform)
.ok_or_else(|| format!("No JRE {jre_key} available for platform: {platform}"))?; .ok_or_else(|| format!("No JRE {jre_key} available for platform: {platform}"))?;
let jre_archive = am.base_dir().join(format!("jre-{}-download.tar.gz", jre_key)); let jre_archive = jre_archive_download_path(am, jre_key, platform_jre.format);
progress(AgentProgressEvent::transfer("jre", 0, platform_jre.size).with_batch( progress(AgentProgressEvent::transfer("jre", 0, platform_jre.size).with_batch(
Some(db_type), Some(db_type),
current, current,
@@ -764,11 +799,11 @@ async fn ensure_jre_from_registry(
let stash = replace_old_jre_dir(&jre_dir)?; let stash = replace_old_jre_dir(&jre_dir)?;
// Persist the stash path *before* extraction so that a crash during // Persist the stash path *before* extraction so that a crash during
// extract_tar_gz (or a process kill) doesn't leave the renamed-stash // archive extraction (or a process kill) doesn't leave the renamed-stash
// directory as an orphan that never gets cleaned up. // directory as an orphan that never gets cleaned up.
persist_pending_jre_cleanup(am, stash.as_ref()).await?; persist_pending_jre_cleanup(am, stash.as_ref()).await?;
extract_tar_gz(&jre_archive, &jre_dir)?; extract_jre_archive(&jre_archive, &jre_dir, platform_jre.format)?;
std::fs::remove_file(&jre_archive).ok(); std::fs::remove_file(&jre_archive).ok();
cleanup_jre_download_cache_after_success(am, jre_key); cleanup_jre_download_cache_after_success(am, jre_key);
@@ -893,6 +928,8 @@ async fn install_agent_driver_from_registry(
if let Some(parent) = target_path.parent() { if let Some(parent) = target_path.parent() {
std::fs::create_dir_all(parent).map_err(|err| format!("Failed to create driver directory: {err}"))?; std::fs::create_dir_all(parent).map_err(|err| format!("Failed to create driver directory: {err}"))?;
} }
let artifact_kind = if is_native_artifact { DriverArtifactKind::Native } else { DriverArtifactKind::Jar };
let download_path = driver_artifact_download_path(&target_path, artifact.format);
progress(AgentProgressEvent::transfer("driver", 0, artifact.size).with_batch( progress(AgentProgressEvent::transfer("driver", 0, artifact.size).with_batch(
Some(db_type), Some(db_type),
current, current,
@@ -905,7 +942,7 @@ async fn install_agent_driver_from_registry(
source, source,
&artifact.url, &artifact.url,
&r2_path_with_cache_buster(&github_url_to_r2_path(&artifact.url, "driver"), &driver.version), &r2_path_with_cache_buster(&github_url_to_r2_path(&artifact.url, "driver"), &driver.version),
&target_path, &download_path,
artifact.size, artifact.size,
Some(CacheIdentity::Driver { db_type, version: &driver.version }), Some(CacheIdentity::Driver { db_type, version: &driver.version }),
Some(db_type), Some(db_type),
@@ -913,16 +950,24 @@ async fn install_agent_driver_from_registry(
total_drivers, total_drivers,
) )
.await?; .await?;
install_downloaded_driver_artifact(
&download_path,
&target_path,
artifact.format,
artifact_kind,
db_type,
&driver.version,
)?;
// Some drivers publish both a native agent and a legacy JAR fallback. Only // Some drivers publish both a native agent and a legacy JAR fallback. Only
// validate the artifact type that was actually installed. // validate the artifact type that was actually installed.
if !is_native_artifact && !am.is_driver_jar_valid(db_type) {
std::fs::remove_file(&target_path).ok();
return Err(format!("Downloaded driver jar is invalid or corrupt: {}", target_path.display()));
}
if is_native_artifact { if is_native_artifact {
mark_executable(&target_path)?; mark_executable(&target_path)?;
std::fs::remove_file(am.driver_jar_path(db_type)).ok(); std::fs::remove_file(am.driver_jar_path(db_type)).ok();
} else { } else {
if !am.is_driver_jar_valid(db_type) {
std::fs::remove_file(&target_path).ok();
return Err(format!("Downloaded driver jar is invalid or corrupt: {}", target_path.display()));
}
std::fs::remove_file(am.driver_native_path(db_type)).ok(); std::fs::remove_file(am.driver_native_path(db_type)).ok();
} }
@@ -947,6 +992,169 @@ async fn install_agent_driver_from_registry(
Ok(()) Ok(())
} }
fn driver_artifact_download_path(target_path: &Path, format: Option<ArtifactFormat>) -> PathBuf {
match format {
Some(ArtifactFormat::TarZstd) => {
let file_name = target_path.file_name().and_then(|name| name.to_str()).unwrap_or("agent");
target_path.with_file_name(format!(".{file_name}.tar.zst"))
}
None => target_path.to_path_buf(),
}
}
fn install_downloaded_driver_artifact(
download_path: &Path,
target_path: &Path,
format: Option<ArtifactFormat>,
artifact_kind: DriverArtifactKind,
db_type: &str,
expected_version: &str,
) -> Result<(), String> {
let Some(format) = format else {
return Ok(());
};
let result = match format {
ArtifactFormat::TarZstd => {
install_driver_from_tar_zstd_package(download_path, target_path, artifact_kind, db_type, expected_version)
}
};
if result.is_ok() {
std::fs::remove_file(download_path).ok();
}
result
}
fn install_driver_from_tar_zstd_package(
package_path: &Path,
target_path: &Path,
expected_kind: DriverArtifactKind,
db_type: &str,
expected_version: &str,
) -> Result<(), String> {
let info = tar_zstd_driver_package_info(package_path)?;
if info.db_type != db_type {
return Err(format!("Driver package contains {}, expected {db_type}", info.db_type));
}
if info.version != expected_version {
return Err(format!(
"Driver package version mismatch for {db_type}: expected {expected_version}, got {}",
info.version
));
}
if info.kind != expected_kind {
return Err(format!(
"Driver package artifact type mismatch for {db_type}: expected {}, got {}",
expected_kind.label(),
info.kind.label()
));
}
let parent = target_path.parent().ok_or_else(|| format!("Invalid driver path: {}", target_path.display()))?;
std::fs::create_dir_all(parent).map_err(|error| format!("Failed to create driver directory: {error}"))?;
let staging_path = parent.join(format!(".agent-package-{}", uuid::Uuid::new_v4()));
let result = extract_tar_zstd_file(package_path, &info.entry_name, &staging_path, info.size).and_then(|_| {
match info.kind {
DriverArtifactKind::Jar if !is_valid_agent_jar(&staging_path) => {
return Err(format!("Packaged driver jar is invalid or corrupt: {}", info.entry_name));
}
DriverArtifactKind::Native => {
validate_native_agent_binary(&staging_path)?;
mark_executable(&staging_path)?;
}
DriverArtifactKind::Jar => {}
}
replace_download(&staging_path, target_path)
});
if result.is_err() {
std::fs::remove_file(&staging_path).ok();
}
result
}
fn read_registry_from_tar_zstd(package_path: &Path) -> Result<AgentRegistry, String> {
const MAX_REGISTRY_BYTES: u64 = 1024 * 1024;
let file = std::fs::File::open(package_path).map_err(|error| format!("Failed to open driver package: {error}"))?;
let decoder = zstd::stream::read::Decoder::new(file)
.map_err(|error| format!("Failed to open zstd driver package: {error}"))?;
let mut archive = tar::Archive::new(decoder);
let entries = archive.entries().map_err(|error| format!("Invalid tar.zst driver package: {error}"))?;
for entry in entries {
let mut entry = entry.map_err(|error| format!("Invalid tar.zst driver package entry: {error}"))?;
let path = entry.path().map_err(|error| format!("Invalid driver package path: {error}"))?;
let name = safe_archive_entry_name(&path)?;
if name != "agent-registry.json" {
continue;
}
if !entry.header().entry_type().is_file() {
return Err("Driver package registry is not a regular file".to_string());
}
if entry.size() > MAX_REGISTRY_BYTES {
return Err("Driver package registry is too large".to_string());
}
let mut json = String::new();
entry.read_to_string(&mut json).map_err(|error| format!("Failed to read driver package registry: {error}"))?;
return serde_json::from_str(&json).map_err(|error| format!("Invalid driver package registry: {error}"));
}
Err("agent-registry.json not found in the driver package".to_string())
}
fn extract_tar_zstd_file(
package_path: &Path,
expected_entry: &str,
destination: &Path,
expected_size: u64,
) -> Result<(), String> {
let file = std::fs::File::open(package_path).map_err(|error| format!("Failed to open driver package: {error}"))?;
let decoder = zstd::stream::read::Decoder::new(file)
.map_err(|error| format!("Failed to open zstd driver package: {error}"))?;
let mut archive = tar::Archive::new(decoder);
let entries = archive.entries().map_err(|error| format!("Invalid tar.zst driver package: {error}"))?;
let mut extracted = false;
for entry in entries {
let mut entry = entry.map_err(|error| format!("Invalid tar.zst driver package entry: {error}"))?;
let path = entry.path().map_err(|error| format!("Invalid driver package path: {error}"))?;
let name = safe_archive_entry_name(&path)?;
if name == "agent-registry.json" || entry.header().entry_type().is_dir() {
continue;
}
if name != expected_entry {
return Err(format!("Unexpected file in driver package: {name}"));
}
if extracted || !entry.header().entry_type().is_file() {
return Err(format!("Invalid driver package entry: {name}"));
}
let mut output =
std::fs::File::create(destination).map_err(|error| format!("Failed to create staged driver: {error}"))?;
let copied = std::io::copy(&mut entry, &mut output)
.map_err(|error| format!("Failed to extract driver package: {error}"))?;
std::io::Write::flush(&mut output).map_err(|error| format!("Failed to flush staged driver: {error}"))?;
if expected_size > 0 && copied != expected_size {
return Err(format!("Packaged driver size mismatch: expected {expected_size} bytes, got {copied} bytes"));
}
extracted = true;
}
if extracted {
Ok(())
} else {
Err(format!("Driver package entry not found: {expected_entry}"))
}
}
fn safe_archive_entry_name(path: &Path) -> Result<String, String> {
let mut parts = Vec::new();
for component in path.components() {
match component {
std::path::Component::Normal(value) => parts.push(value.to_string_lossy().into_owned()),
std::path::Component::CurDir => {}
_ => return Err(format!("Unsafe driver package path: {}", path.display())),
}
}
if parts.is_empty() {
return Err("Driver package contains an empty path".to_string());
}
Ok(parts.join("/"))
}
fn agent_registry_driver<'a>( fn agent_registry_driver<'a>(
registry: &'a AgentRegistry, registry: &'a AgentRegistry,
db_type: &str, db_type: &str,
@@ -1416,10 +1624,181 @@ pub struct OfflineImportPlan {
pub includes_jre: bool, pub includes_jre: bool,
} }
type OfflineJreEntry = (String, String); type OfflineJreEntry = (String, String, Option<ArtifactFormat>);
type OfflineDriverEntry = (String, String, bool); type OfflineDriverEntry = (String, String, bool);
type OfflineArchiveEntries = (Vec<OfflineJreEntry>, Vec<OfflineDriverEntry>); type OfflineArchiveEntries = (Vec<OfflineJreEntry>, Vec<OfflineDriverEntry>);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum DriverArtifactKind {
Jar,
Native,
}
impl DriverArtifactKind {
fn label(self) -> &'static str {
match self {
Self::Jar => "jar",
Self::Native => "native",
}
}
}
#[derive(Debug, Clone)]
struct TarZstdDriverPackageInfo {
db_type: String,
version: String,
jre: String,
kind: DriverArtifactKind,
entry_name: String,
size: u64,
}
fn inspect_tar_zstd_driver_package(package_path: &Path) -> Result<OfflineImportPlan, String> {
let info = tar_zstd_driver_package_info(package_path)?;
Ok(OfflineImportPlan { driver_keys: vec![info.db_type], includes_jre: false })
}
async fn import_tar_zstd_driver_package(
am: &AgentManager,
package_path: &Path,
progress: impl Fn(OfflineImportProgress),
) -> Result<OfflineImportResult, String> {
let _installation_guard = am.installation_operation_lock.write().await;
let info = tar_zstd_driver_package_info(package_path)?;
let mut result =
OfflineImportResult { jre_installed: Vec::new(), drivers_installed: Vec::new(), drivers_skipped: Vec::new() };
if let Some(installed) = am.load_state().installed_drivers.get(&info.db_type) {
if installed.version != "0.1.0-local"
&& installed.version != "local"
&& !crate::update::is_newer_version(&info.version, &installed.version)
{
result.drivers_skipped.push(info.db_type);
return Ok(result);
}
}
progress(OfflineImportProgress {
step: "driver".to_string(),
current: 1,
total: 1,
label: agent_catalog::label_for_key(&info.db_type).unwrap_or(&info.db_type).to_string(),
db_type: Some(info.db_type.clone()),
});
let target_path = match info.kind {
DriverArtifactKind::Jar => am.driver_jar_path(&info.db_type),
DriverArtifactKind::Native => am.driver_native_path(&info.db_type),
};
install_driver_from_tar_zstd_package(package_path, &target_path, info.kind, &info.db_type, &info.version)?;
match info.kind {
DriverArtifactKind::Jar => {
std::fs::remove_file(am.driver_native_path(&info.db_type)).ok();
}
DriverArtifactKind::Native => {
std::fs::remove_file(am.driver_jar_path(&info.db_type)).ok();
}
}
am.mutate_state(|state| {
state.installed_drivers.insert(
info.db_type.clone(),
InstalledDriver {
version: info.version.clone(),
installed_at: chrono::Utc::now().to_rfc3339(),
jre: info.jre.clone(),
},
);
})?;
am.stop_daemon_by_key(&info.db_type).await;
result.drivers_installed.push(info.db_type);
Ok(result)
}
fn tar_zstd_driver_package_info(package_path: &Path) -> Result<TarZstdDriverPackageInfo, String> {
let registry = read_registry_from_tar_zstd(package_path)?;
if registry.drivers.len() != 1 {
return Err("A tar.zst driver package must contain exactly one driver".to_string());
}
let (db_type, driver) = registry.drivers.iter().next().expect("checked one driver");
validate_offline_driver_key(db_type)?;
let platform = AgentManager::current_platform();
let native_artifact = driver.native.get(platform);
let jar_artifact = driver.jar.as_ref();
let (kind, artifact) = match (native_artifact, jar_artifact) {
(Some(_), Some(_)) => {
return Err("A tar.zst driver package must contain exactly one driver artifact".to_string());
}
(Some(artifact), None) => (DriverArtifactKind::Native, artifact),
(None, Some(artifact)) => (DriverArtifactKind::Jar, artifact),
(None, None) if !driver.native.is_empty() => {
return Err(format!("Driver package does not support platform: {platform}"));
}
(None, None) => return Err("A tar.zst driver package contains no driver artifact".to_string()),
};
if artifact.format.is_some() {
return Err("Nested driver packages are not supported".to_string());
}
let artifact_filename = artifact
.url
.rsplit('/')
.next()
.filter(|name| !name.is_empty())
.ok_or_else(|| format!("Invalid packaged driver URL: {}", artifact.url))?;
let entry_name = format!("drivers/{artifact_filename}");
validate_tar_zstd_package_entries(package_path, &entry_name, artifact.size)?;
Ok(TarZstdDriverPackageInfo {
db_type: db_type.clone(),
version: driver.version.clone(),
jre: driver.jre.clone(),
kind,
entry_name,
size: artifact.size,
})
}
fn validate_tar_zstd_package_entries(
package_path: &Path,
expected_entry: &str,
expected_size: u64,
) -> Result<(), String> {
let file = std::fs::File::open(package_path).map_err(|error| format!("Failed to open driver package: {error}"))?;
let decoder = zstd::stream::read::Decoder::new(file)
.map_err(|error| format!("Failed to open zstd driver package: {error}"))?;
let mut archive = tar::Archive::new(decoder);
let entries = archive.entries().map_err(|error| format!("Invalid tar.zst driver package: {error}"))?;
let mut registry_seen = false;
let mut driver_seen = false;
for entry in entries {
let entry = entry.map_err(|error| format!("Invalid tar.zst driver package entry: {error}"))?;
let path = entry.path().map_err(|error| format!("Invalid driver package path: {error}"))?;
let name = safe_archive_entry_name(&path)?;
if entry.header().entry_type().is_dir() {
continue;
}
if !entry.header().entry_type().is_file() {
return Err(format!("Driver package contains a non-regular entry: {name}"));
}
match name.as_str() {
"agent-registry.json" if !registry_seen => registry_seen = true,
value if value == expected_entry && !driver_seen => {
if expected_size > 0 && entry.size() != expected_size {
return Err(format!(
"Packaged driver size mismatch: expected {expected_size} bytes, got {} bytes",
entry.size()
));
}
driver_seen = true;
}
_ => return Err(format!("Unexpected file in driver package: {name}")),
}
}
if !registry_seen {
return Err("agent-registry.json not found in the driver package".to_string());
}
if !driver_seen {
return Err(format!("Driver package entry not found: {expected_entry}"));
}
Ok(())
}
pub fn inspect_offline_zip(zip_path: &Path) -> Result<OfflineImportPlan, String> { pub fn inspect_offline_zip(zip_path: &Path) -> Result<OfflineImportPlan, String> {
let file = std::fs::File::open(zip_path).map_err(|e| format!("Failed to open ZIP file: {e}"))?; let file = std::fs::File::open(zip_path).map_err(|e| format!("Failed to open ZIP file: {e}"))?;
let mut archive = zip::ZipArchive::new(file).map_err(|e| format!("Invalid ZIP file: {e}"))?; let mut archive = zip::ZipArchive::new(file).map_err(|e| format!("Invalid ZIP file: {e}"))?;
@@ -1461,7 +1840,7 @@ pub async fn import_offline_zip(
validate_offline_driver_entries(am, &mut archive, &driver_entries)?; validate_offline_driver_entries(am, &mut archive, &driver_entries)?;
let mut current: u32 = 0; let mut current: u32 = 0;
for (jre_key, entry_name) in &jre_entries { for (jre_key, entry_name, format) in &jre_entries {
current += 1; current += 1;
let jre_version = registry.resolve_jre(jre_key).map(|j| j.version.clone()); let jre_version = registry.resolve_jre(jre_key).map(|j| j.version.clone());
let existing_version = local_state.jre_versions.get(jre_key); let existing_version = local_state.jre_versions.get(jre_key);
@@ -1478,7 +1857,7 @@ pub async fn import_offline_zip(
}); });
let mut entry = archive.by_name(entry_name).map_err(|e| format!("Failed to read {entry_name}: {e}"))?; let mut entry = archive.by_name(entry_name).map_err(|e| format!("Failed to read {entry_name}: {e}"))?;
let tmp_archive = am.base_dir().join(format!("jre-offline-{jre_key}.tar.gz")); let tmp_archive = am.base_dir().join(format!("jre-offline-{jre_key}{}", jre_archive_suffix(*format)));
{ {
let mut out = let mut out =
std::fs::File::create(&tmp_archive).map_err(|e| format!("Failed to create temp file: {e}"))?; std::fs::File::create(&tmp_archive).map_err(|e| format!("Failed to create temp file: {e}"))?;
@@ -1487,7 +1866,7 @@ pub async fn import_offline_zip(
let jre_dir = am.jre_dir(jre_key); let jre_dir = am.jre_dir(jre_key);
let staging_dir = am.base_dir().join(format!(".jre-offline-import-{}", uuid::Uuid::new_v4())); let staging_dir = am.base_dir().join(format!(".jre-offline-import-{}", uuid::Uuid::new_v4()));
if let Err(error) = extract_tar_gz(&tmp_archive, &staging_dir) { if let Err(error) = extract_jre_archive(&tmp_archive, &staging_dir, *format) {
std::fs::remove_dir_all(&staging_dir).ok(); std::fs::remove_dir_all(&staging_dir).ok();
std::fs::remove_file(&tmp_archive).ok(); std::fs::remove_file(&tmp_archive).ok();
return Err(error); return Err(error);
@@ -1599,7 +1978,7 @@ fn collect_offline_entries(
registry: &AgentRegistry, registry: &AgentRegistry,
) -> Result<OfflineArchiveEntries, String> { ) -> Result<OfflineArchiveEntries, String> {
let platform = AgentManager::current_platform(); let platform = AgentManager::current_platform();
let mut jre_entries = Vec::new(); let mut jres = std::collections::BTreeMap::<String, (String, Option<ArtifactFormat>)>::new();
let mut drivers = std::collections::BTreeMap::<String, (String, bool)>::new(); let mut drivers = std::collections::BTreeMap::<String, (String, bool)>::new();
for index in 0..archive.len() { for index in 0..archive.len() {
@@ -1608,11 +1987,21 @@ fn collect_offline_entries(
return Err(format!("Offline package contains an unsafe path: {}", entry.name())); return Err(format!("Offline package contains an unsafe path: {}", entry.name()));
}; };
let name = path.to_string_lossy().replace('\\', "/"); let name = path.to_string_lossy().replace('\\', "/");
if name.starts_with("jre/") && name.ends_with(".tar.gz") && name.contains(platform) { if name.starts_with("jre/") && name.contains(platform) {
let jre_format = if name.ends_with(".tar.zst") {
Some(ArtifactFormat::TarZstd)
} else if name.ends_with(".tar.gz") {
None
} else {
continue;
};
let jre_key = extract_jre_key_from_filename(&name) let jre_key = extract_jre_key_from_filename(&name)
.ok_or_else(|| format!("Invalid JRE filename in offline package: {name}"))?; .ok_or_else(|| format!("Invalid JRE filename in offline package: {name}"))?;
validate_offline_identifier(&jre_key, "JRE")?; validate_offline_identifier(&jre_key, "JRE")?;
jre_entries.push((jre_key, name)); let replace = !jres.contains_key(&jre_key) || jre_format == Some(ArtifactFormat::TarZstd);
if replace {
jres.insert(jre_key, (name, jre_format));
}
} else if name.starts_with("drivers/") && name.ends_with(".jar") { } else if name.starts_with("drivers/") && name.ends_with(".jar") {
let db_type = db_type_for_jar_offline_entry(registry, &name) let db_type = db_type_for_jar_offline_entry(registry, &name)
.or_else(|| extract_db_type_from_filename(&name)) .or_else(|| extract_db_type_from_filename(&name))
@@ -1629,7 +2018,10 @@ fn collect_offline_entries(
} }
} }
Ok((jre_entries, drivers.into_iter().map(|(db_type, (name, is_native))| (db_type, name, is_native)).collect())) Ok((
jres.into_iter().map(|(jre_key, (name, format))| (jre_key, name, format)).collect(),
drivers.into_iter().map(|(db_type, (name, is_native))| (db_type, name, is_native)).collect(),
))
} }
fn validate_offline_driver_entries( fn validate_offline_driver_entries(
@@ -1723,7 +2115,36 @@ fn db_type_for_jar_offline_entry(registry: &AgentRegistry, name: &str) -> Option
}) })
} }
fn jre_archive_suffix(format: Option<ArtifactFormat>) -> &'static str {
match format {
Some(ArtifactFormat::TarZstd) => ".tar.zst",
None => ".tar.gz",
}
}
fn jre_archive_download_path(am: &AgentManager, jre_key: &str, format: Option<ArtifactFormat>) -> PathBuf {
am.base_dir().join(format!("jre-{jre_key}-download{}", jre_archive_suffix(format)))
}
fn extract_jre_archive(archive: &Path, dest: &Path, format: Option<ArtifactFormat>) -> Result<(), String> {
match format {
Some(ArtifactFormat::TarZstd) => {
let file = std::fs::File::open(archive).map_err(|e| format!("Failed to open JRE archive: {e}"))?;
let decoder =
zstd::stream::read::Decoder::new(file).map_err(|e| format!("Failed to open zstd JRE archive: {e}"))?;
extract_jre_tar(tar::Archive::new(decoder), dest)
}
None => extract_tar_gz(archive, dest),
}
}
fn extract_tar_gz(archive: &Path, dest: &Path) -> Result<(), String> { fn extract_tar_gz(archive: &Path, dest: &Path) -> Result<(), String> {
let file = std::fs::File::open(archive).map_err(|e| format!("Failed to open JRE archive: {e}"))?;
let decoder = flate2::read::GzDecoder::new(file);
extract_jre_tar(tar::Archive::new(decoder), dest)
}
fn extract_jre_tar<R: Read>(mut archive: tar::Archive<R>, dest: &Path) -> Result<(), String> {
let parent = dest.parent().ok_or_else(|| format!("Invalid JRE destination: {}", dest.display()))?; let parent = dest.parent().ok_or_else(|| format!("Invalid JRE destination: {}", dest.display()))?;
std::fs::create_dir_all(parent).map_err(|e| format!("Failed to create JRE directory: {e}"))?; std::fs::create_dir_all(parent).map_err(|e| format!("Failed to create JRE directory: {e}"))?;
@@ -1731,9 +2152,7 @@ fn extract_tar_gz(archive: &Path, dest: &Path) -> Result<(), String> {
.prefix(".jre-extract-") .prefix(".jre-extract-")
.tempdir_in(parent) .tempdir_in(parent)
.map_err(|e| format!("Failed to create JRE extraction directory: {e}"))?; .map_err(|e| format!("Failed to create JRE extraction directory: {e}"))?;
let file = std::fs::File::open(archive).map_err(|e| format!("Failed to open JRE archive: {e}"))?; archive.unpack(staging.path()).map_err(|e| format!("Failed to extract JRE archive: {e}"))?;
let decoder = flate2::read::GzDecoder::new(file);
tar::Archive::new(decoder).unpack(staging.path()).map_err(|e| format!("Failed to extract JRE archive: {e}"))?;
let mut roots = std::fs::read_dir(staging.path()) let mut roots = std::fs::read_dir(staging.path())
.map_err(|e| format!("Failed to inspect extracted JRE archive: {e}"))? .map_err(|e| format!("Failed to inspect extracted JRE archive: {e}"))?
@@ -2090,7 +2509,7 @@ mod agent_download_url_tests {
#[cfg(test)] #[cfg(test)]
mod agent_registry_install_tests { mod agent_registry_install_tests {
use super::*; use super::*;
use crate::agent_manager::{ArtifactInfo, DriverInfo, InstalledDriver, JavaRuntimeConfig, JreInfo}; use crate::agent_manager::{ArtifactFormat, ArtifactInfo, DriverInfo, InstalledDriver, JavaRuntimeConfig, JreInfo};
fn test_manager(name: &str) -> AgentManager { fn test_manager(name: &str) -> AgentManager {
let dir = std::env::temp_dir().join(format!("dbx-agent-registry-install-{name}-{}", uuid::Uuid::new_v4())); let dir = std::env::temp_dir().join(format!("dbx-agent-registry-install-{name}-{}", uuid::Uuid::new_v4()));
@@ -2124,10 +2543,11 @@ mod agent_registry_install_tests {
jar: Some(ArtifactInfo { jar: Some(ArtifactInfo {
url: format!("https://example.com/dbx-agent-{db_type}-legacy-placeholder.jar"), url: format!("https://example.com/dbx-agent-{db_type}-legacy-placeholder.jar"),
size: 0, size: 0,
format: None,
}), }),
native: [( native: [(
AgentManager::current_platform().to_string(), AgentManager::current_platform().to_string(),
ArtifactInfo { url: native_url.to_string(), size: native_size }, ArtifactInfo { url: native_url.to_string(), size: native_size, format: None },
)] )]
.into_iter() .into_iter()
.collect(), .collect(),
@@ -2145,7 +2565,7 @@ mod agent_registry_install_tests {
label: db_type.to_string(), label: db_type.to_string(),
min_app_version: "0.1.0".to_string(), min_app_version: "0.1.0".to_string(),
jre: DEFAULT_JRE_KEY.to_string(), jre: DEFAULT_JRE_KEY.to_string(),
jar: Some(ArtifactInfo { url: url.to_string(), size }), jar: Some(ArtifactInfo { url: url.to_string(), size, format: None }),
native: std::collections::HashMap::new(), native: std::collections::HashMap::new(),
}, },
); );
@@ -2180,6 +2600,110 @@ mod agent_registry_install_tests {
cache_path cache_path
} }
fn current_platform_native_binary() -> Vec<u8> {
if cfg!(windows) {
let mut bytes = vec![0_u8; 0x48];
bytes[..2].copy_from_slice(b"MZ");
bytes[0x3c..0x40].copy_from_slice(&(0x40_u32).to_le_bytes());
bytes[0x40..0x44].copy_from_slice(b"PE\0\0");
let machine = if cfg!(target_arch = "aarch64") { 0xaa64_u16 } else { 0x8664_u16 };
bytes[0x44..0x46].copy_from_slice(&machine.to_le_bytes());
bytes
} else if cfg!(target_os = "linux") {
let mut bytes = vec![0_u8; 20];
bytes[..4].copy_from_slice(b"\x7fELF");
bytes[4] = 2;
bytes[5] = 1;
let machine = if cfg!(target_arch = "aarch64") { 183_u16 } else { 62_u16 };
bytes[18..20].copy_from_slice(&machine.to_le_bytes());
bytes
} else if cfg!(target_os = "macos") {
let mut bytes = vec![0xcf, 0xfa, 0xed, 0xfe];
let cpu_type = if cfg!(target_arch = "aarch64") { 0x0100_000c_u32 } else { 0x0100_0007_u32 };
bytes.extend_from_slice(&cpu_type.to_le_bytes());
bytes
} else {
Vec::new()
}
}
fn test_agent_jar() -> Vec<u8> {
let mut bytes = std::io::Cursor::new(Vec::new());
{
use std::io::Write;
let mut archive = zip::ZipWriter::new(&mut bytes);
archive.start_file("META-INF/MANIFEST.MF", zip::write::SimpleFileOptions::default()).unwrap();
archive.write_all(b"Manifest-Version: 1.0\nMain-Class: com.dbx.Agent\n").unwrap();
archive.finish().unwrap();
}
bytes.into_inner()
}
fn build_tar_zstd_driver_package(
db_type: &str,
version: &str,
kind: DriverArtifactKind,
driver_bytes: &[u8],
) -> Vec<u8> {
let platform = AgentManager::current_platform();
let (filename, artifact) = match kind {
DriverArtifactKind::Jar => (
format!("dbx-agent-{db_type}-{version}.jar"),
serde_json::json!({
"jar": {
"url": format!("dbx-agent-{db_type}-{version}.jar"),
"size": driver_bytes.len()
}
}),
),
DriverArtifactKind::Native => {
let extension = if platform.starts_with("windows-") { ".exe" } else { "" };
let filename = format!("dbx-agent-{db_type}-{version}-{platform}{extension}");
(
filename.clone(),
serde_json::json!({
"native": {
platform: {
"url": filename,
"size": driver_bytes.len()
}
}
}),
)
}
};
let mut driver = serde_json::json!({
"version": version,
"label": db_type,
"min_app_version": "0.6.0",
"jre": DEFAULT_JRE_KEY
});
driver.as_object_mut().unwrap().extend(artifact.as_object().unwrap().clone());
let registry = serde_json::json!({
"jres": {},
"drivers": {
db_type: driver
}
});
let registry_bytes = registry.to_string().into_bytes();
let encoder = zstd::stream::write::Encoder::new(Vec::new(), 3).unwrap();
let mut archive = tar::Builder::new(encoder);
let mut registry_header = tar::Header::new_gnu();
registry_header.set_size(registry_bytes.len() as u64);
registry_header.set_mode(0o644);
registry_header.set_cksum();
archive.append_data(&mut registry_header, "agent-registry.json", registry_bytes.as_slice()).unwrap();
let mut driver_header = tar::Header::new_gnu();
driver_header.set_size(driver_bytes.len() as u64);
driver_header.set_mode(if kind == DriverArtifactKind::Native { 0o755 } else { 0o644 });
driver_header.set_cksum();
archive.append_data(&mut driver_header, format!("drivers/{filename}"), driver_bytes).unwrap();
archive.into_inner().unwrap().finish().unwrap()
}
#[test] #[test]
fn managed_jre_content_revision_triggers_install() { fn managed_jre_content_revision_triggers_install() {
let manager = test_manager("jre-content-revision"); let manager = test_manager("jre-content-revision");
@@ -2208,7 +2732,7 @@ mod agent_registry_install_tests {
version: version.to_string(), version: version.to_string(),
platforms: [( platforms: [(
AgentManager::current_platform().to_string(), AgentManager::current_platform().to_string(),
ArtifactInfo { url: url.to_string(), size }, ArtifactInfo { url: url.to_string(), size, format: None },
)] )]
.into_iter() .into_iter()
.collect(), .collect(),
@@ -2234,8 +2758,29 @@ mod agent_registry_install_tests {
builder.into_inner().unwrap().finish().unwrap() builder.into_inner().unwrap().finish().unwrap()
} }
fn write_cached_jre_download(am: &AgentManager, jre_key: &str, version: &str, url: &str, archive: &[u8]) { fn build_zstd_jre_archive(am: &AgentManager, jre_key: &str) -> Vec<u8> {
let dest = am.base_dir().join(format!("jre-{jre_key}-download.tar.gz")); let archive_root = am.base_dir().join("jre-zstd-test-archive");
let payload = archive_root.join("payload");
let java_path = am.jre_java_path(jre_key);
let relative_java_path = java_path.strip_prefix(am.jre_dir(jre_key)).unwrap();
let java_path = payload.join(relative_java_path);
std::fs::create_dir_all(java_path.parent().unwrap()).unwrap();
std::fs::write(java_path, b"java").unwrap();
let encoder = zstd::stream::write::Encoder::new(Vec::new(), 3).unwrap();
let mut builder = tar::Builder::new(encoder);
builder.append_dir_all("payload", &payload).unwrap();
builder.into_inner().unwrap().finish().unwrap()
}
fn write_cached_jre_download(
am: &AgentManager,
jre_key: &str,
version: &str,
url: &str,
format: Option<ArtifactFormat>,
archive: &[u8],
) {
let dest = jre_archive_download_path(am, jre_key, format);
let cache_path = cached_download_path( let cache_path = cached_download_path(
am, am,
url, url,
@@ -2284,6 +2829,81 @@ mod agent_registry_install_tests {
.any(|event| event.step == "done" && event.db_type.as_deref() == Some(db_type))); .any(|event| event.step == "done" && event.db_type.as_deref() == Some(db_type)));
} }
#[tokio::test]
async fn registry_install_extracts_tar_zstd_native_driver_package() {
let manager = test_manager("tar-zstd-native-package");
let db_type = "duckdb";
let version = "0.1.0";
let package_url = "https://example.com/dbx-agent-duckdb.tar.zst";
let native_bytes = current_platform_native_binary();
let package_bytes = build_tar_zstd_driver_package(db_type, version, DriverArtifactKind::Native, &native_bytes);
let mut registry =
registry_with_native_and_legacy_jar(db_type, version, package_url, package_bytes.len() as u64);
registry.drivers.get_mut(db_type).unwrap().native.get_mut(AgentManager::current_platform()).unwrap().format =
Some(ArtifactFormat::TarZstd);
let native_path = manager.driver_native_path(db_type);
let package_path = driver_artifact_download_path(&native_path, Some(ArtifactFormat::TarZstd));
let cache_path =
write_cached_driver_download(&manager, db_type, version, package_url, &package_path, &package_bytes);
let progress = |_| {};
install_agent_driver_from_registry(
&manager,
&registry,
DownloadSource::Official,
db_type,
&progress,
None,
None,
)
.await
.unwrap();
assert_eq!(std::fs::read(&native_path).unwrap(), native_bytes);
assert!(!package_path.exists());
assert!(!cache_path.exists());
assert_eq!(manager.load_state().installed_drivers[db_type].version, version);
}
#[tokio::test]
async fn registry_install_extracts_tar_zstd_java_driver_package() {
let manager = test_manager("tar-zstd-java-package");
let db_type = "dameng";
let version = "0.2.0";
let package_url = "https://example.com/dbx-agent-dameng.tar.zst";
let jar_bytes = test_agent_jar();
let package_bytes = build_tar_zstd_driver_package(db_type, version, DriverArtifactKind::Jar, &jar_bytes);
let mut registry = registry_with_jar(db_type, version, package_url, package_bytes.len() as u64);
registry.drivers.get_mut(db_type).unwrap().jar.as_mut().unwrap().format = Some(ArtifactFormat::TarZstd);
manager
.mutate_state(|state| {
state.java_runtime = JavaRuntimeConfig { mode: JavaRuntimeMode::System, custom_java_path: None };
})
.unwrap();
let jar_path = manager.driver_jar_path(db_type);
let package_path = driver_artifact_download_path(&jar_path, Some(ArtifactFormat::TarZstd));
let cache_path =
write_cached_driver_download(&manager, db_type, version, package_url, &package_path, &package_bytes);
let progress = |_| {};
install_agent_driver_from_registry(
&manager,
&registry,
DownloadSource::Official,
db_type,
&progress,
None,
None,
)
.await
.unwrap();
assert_eq!(std::fs::read(&jar_path).unwrap(), jar_bytes);
assert!(!package_path.exists());
assert!(!cache_path.exists());
assert_eq!(manager.load_state().installed_drivers[db_type].version, version);
}
#[tokio::test] #[tokio::test]
async fn batch_upgrade_preserves_successful_state_and_reports_independent_failure() { async fn batch_upgrade_preserves_successful_state_and_reports_independent_failure() {
let manager = test_manager("batch-upgrade"); let manager = test_manager("batch-upgrade");
@@ -2367,7 +2987,7 @@ mod agent_registry_install_tests {
let events = std::sync::Mutex::new(Vec::new()); let events = std::sync::Mutex::new(Vec::new());
let progress = |event| events.lock().unwrap().push(event); let progress = |event| events.lock().unwrap().push(event);
write_cached_jre_download(&manager, jre_key, version, url, &archive); write_cached_jre_download(&manager, jre_key, version, url, None, &archive);
ensure_jre_from_registry( ensure_jre_from_registry(
&manager, &manager,
&registry, &registry,
@@ -2383,7 +3003,7 @@ mod agent_registry_install_tests {
// The successful install cleans its cache. Re-add it so the old // The successful install cleans its cache. Re-add it so the old
// implementation fails deterministically by consuming it again. // implementation fails deterministically by consuming it again.
write_cached_jre_download(&manager, jre_key, version, url, &archive); write_cached_jre_download(&manager, jre_key, version, url, None, &archive);
ensure_jre_from_registry( ensure_jre_from_registry(
&manager, &manager,
&registry, &registry,
@@ -2410,6 +3030,26 @@ mod agent_registry_install_tests {
.any(|event| event.step == "jre" && event.db_type.as_deref() == Some("dameng"))); .any(|event| event.step == "jre" && event.db_type.as_deref() == Some("dameng")));
} }
#[tokio::test]
async fn managed_jre_install_extracts_tar_zstd_archive() {
let manager = test_manager("managed-zstd-jre");
let jre_key = DEFAULT_JRE_KEY;
let version = "21.0.12";
let url = "https://example.com/dbx-jre.tar.zst";
let archive = build_zstd_jre_archive(&manager, jre_key);
let mut registry = registry_with_jre(jre_key, version, url, archive.len() as u64);
registry.jres.get_mut(jre_key).unwrap().platforms.get_mut(AgentManager::current_platform()).unwrap().format =
Some(ArtifactFormat::TarZstd);
write_cached_jre_download(&manager, jre_key, version, url, Some(ArtifactFormat::TarZstd), &archive);
ensure_jre_from_registry(&manager, &registry, DownloadSource::Official, jre_key, "dameng", &|_| {}, None, None)
.await
.unwrap();
assert!(manager.is_jre_installed(jre_key));
assert_eq!(manager.load_state().jre_versions[jre_key], version);
}
#[tokio::test] #[tokio::test]
async fn stash_is_recorded_before_jre_extraction() { async fn stash_is_recorded_before_jre_extraction() {
let manager = test_manager("stash-before-extract"); let manager = test_manager("stash-before-extract");
+55 -326
View File
@@ -4,8 +4,6 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc; use std::sync::Arc;
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
use tokio::sync::{watch, Mutex, RwLock}; use tokio::sync::{watch, Mutex, RwLock};
#[cfg(feature = "duckdb-bundled")]
use tokio::task::JoinHandle;
use mysql_async::prelude::Queryable; use mysql_async::prelude::Queryable;
use mysql_async::Row as MysqlRow; use mysql_async::Row as MysqlRow;
@@ -42,21 +40,16 @@ const ACCESS_AGENT_CONNECT_TIMEOUT_SECS: u64 = 30;
const POOL_CLOSE_TIMEOUT_SECS: u64 = 3; const POOL_CLOSE_TIMEOUT_SECS: u64 = 3;
const HEALTH_CHECK_POOL_ACQUIRE_TIMEOUT: Duration = Duration::from_millis(500); const HEALTH_CHECK_POOL_ACQUIRE_TIMEOUT: Duration = Duration::from_millis(500);
#[cfg(feature = "duckdb-bundled")]
mod duckdb_types { mod duckdb_types {
#[cfg(feature = "duckdb-sidecar")]
use std::sync::Arc; use std::sync::Arc;
pub type DuckDbHandle = Arc<crate::db::duckdb_driver::DuckDbConnection>; #[cfg(feature = "duckdb-sidecar")]
pub type DuckDbWorkerHandle = Arc<crate::db::duckdb_worker_process::DuckDbWorkerClient>; pub type DuckDbWorkerHandle = Arc<crate::db::duckdb_worker_process::DuckDbWorkerClient>;
pub type ExternalTabularHandle = Arc<crate::external::ExternalPool>; #[cfg(not(feature = "duckdb-sidecar"))]
}
#[cfg(not(feature = "duckdb-bundled"))]
mod duckdb_types {
pub type DuckDbHandle = ();
pub type DuckDbWorkerHandle = (); pub type DuckDbWorkerHandle = ();
pub type ExternalTabularHandle = ();
} }
use duckdb_types::{DuckDbHandle, DuckDbWorkerHandle, ExternalTabularHandle}; use duckdb_types::DuckDbWorkerHandle;
#[derive(Debug, Clone, Copy, PartialEq)] #[derive(Debug, Clone, Copy, PartialEq)]
pub enum MysqlMode { pub enum MysqlMode {
@@ -100,7 +93,6 @@ pub enum PoolKind {
Turso(db::turso_driver::TursoClient), Turso(db::turso_driver::TursoClient),
CloudflareD1(db::cloudflare_d1_driver::CloudflareD1Client), CloudflareD1(db::cloudflare_d1_driver::CloudflareD1Client),
Redis(db::redis_driver::RedisConnection), Redis(db::redis_driver::RedisConnection),
DuckDb(DuckDbHandle),
DuckDbWorker(DuckDbWorkerHandle), DuckDbWorker(DuckDbWorkerHandle),
MongoDb(mongodb::Client), MongoDb(mongodb::Client),
ClickHouse(db::clickhouse_driver::ChClient), ClickHouse(db::clickhouse_driver::ChClient),
@@ -110,7 +102,6 @@ pub enum PoolKind {
VectorDb(db::vector_driver::VectorClient), VectorDb(db::vector_driver::VectorClient),
InfluxDb(db::influxdb_driver::InfluxdbClient), InfluxDb(db::influxdb_driver::InfluxdbClient),
Agent(Arc<tokio::sync::Mutex<db::agent_driver::AgentDriverClient>>), Agent(Arc<tokio::sync::Mutex<db::agent_driver::AgentDriverClient>>),
ExternalTabular(ExternalTabularHandle),
ExternalDriver { ExternalDriver {
driver_id: String, driver_id: String,
config: Arc<ConnectionConfig>, config: Arc<ConnectionConfig>,
@@ -690,41 +681,44 @@ impl AppState {
} }
} }
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
async fn create_duckdb_pool(&self, config: &ConnectionConfig) -> Result<PoolKind, String> { pub async fn create_duckdb_pool(&self, config: &ConnectionConfig) -> Result<PoolKind, String> {
if self.duckdb_worker_process_isolation.load(Ordering::Relaxed) { let attached_databases = config
let attached_databases = config .attached_databases
.attached_databases .iter()
.iter() .map(|attached| crate::models::connection::AttachedDatabaseConfig {
.map(|attached| crate::models::connection::AttachedDatabaseConfig { name: attached.name.clone(),
name: attached.name.clone(), path: expand_tilde(&attached.path),
path: expand_tilde(&attached.path), })
}) .collect();
.collect(); let path = expand_tilde(&config.host);
let client = db::duckdb_worker_process::DuckDbWorkerClient::open_with_process_limit( let init_script = config.init_script.clone();
expand_tilde(&config.host), let process_limit = self.duckdb_worker_max_processes.load(Ordering::Relaxed);
let installed_driver = self.agent_manager.driver_native_path("duckdb");
let has_driver_override =
std::env::var_os(db::duckdb_worker_process::DUCKDB_DRIVER_PATH_ENV).is_some_and(|value| !value.is_empty());
let client = if installed_driver.is_file() && !has_driver_override {
db::duckdb_worker_process::DuckDbWorkerClient::open_with_executable_and_process_limit(
installed_driver,
path,
attached_databases, attached_databases,
config.init_script.clone(), init_script,
self.duckdb_worker_max_processes.load(Ordering::Relaxed), process_limit,
) )
.await?; .await?
Ok(PoolKind::DuckDbWorker(Arc::new(client)))
} else { } else {
let con = db::duckdb_driver::connect_path(&expand_tilde(&config.host))?; db::duckdb_worker_process::DuckDbWorkerClient::open_with_process_limit(
{ path,
let locked = con.lock().map_err(|e| e.to_string())?; attached_databases,
for attached in &config.attached_databases { init_script,
crate::schema::duckdb_attach_database(&locked, &attached.name, &expand_tilde(&attached.path))?; process_limit,
} )
if let Some(script) = config.init_script.as_deref() { .await?
db::duckdb_driver::run_init_script(&locked, script)?; };
} Ok(PoolKind::DuckDbWorker(Arc::new(client)))
}
Ok(PoolKind::DuckDb(con))
}
} }
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
pub async fn test_duckdb_connection_config(&self, config: &ConnectionConfig) -> Result<(), String> { pub async fn test_duckdb_connection_config(&self, config: &ConnectionConfig) -> Result<(), String> {
// Test the submitted form as a fresh session so unsaved ATTACH/init // Test the submitted form as a fresh session so unsaved ATTACH/init
// changes cannot be masked by a pool created from older settings. // changes cannot be masked by a pool created from older settings.
@@ -1315,11 +1309,11 @@ impl AppState {
}; };
PoolKind::Redis(con) PoolKind::Redis(con)
} }
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
DatabaseType::DuckDb => self.create_duckdb_pool(&db_config).await?, DatabaseType::DuckDb => self.create_duckdb_pool(&db_config).await?,
#[cfg(not(feature = "duckdb-bundled"))] #[cfg(not(feature = "duckdb-sidecar"))]
DatabaseType::DuckDb => { DatabaseType::DuckDb => {
return Err("DuckDB support is not compiled in this build. Rebuild with default features.".to_string()); return Err("DuckDB support is not compiled in this build.".to_string());
} }
DatabaseType::MongoDb => { DatabaseType::MongoDb => {
if mongo_uses_legacy_driver(&db_config) { if mongo_uses_legacy_driver(&db_config) {
@@ -2342,9 +2336,7 @@ impl AppState {
} }
} }
PoolKind::Sqlite(_) PoolKind::Sqlite(_)
| PoolKind::DuckDb(_)
| PoolKind::DuckDbWorker(_) | PoolKind::DuckDbWorker(_)
| PoolKind::ExternalTabular(_)
| PoolKind::ExternalDriver { .. } | PoolKind::ExternalDriver { .. }
| PoolKind::MessageQueue | PoolKind::MessageQueue
| PoolKind::Nacos => false, | PoolKind::Nacos => false,
@@ -2567,123 +2559,10 @@ impl AppState {
} }
} }
#[cfg(feature = "duckdb-bundled")]
async fn remove_pool_by_key_detached(&self, pool_key: &str) -> bool {
self.stop_keepalive_task(pool_key).await;
self.pool_activity.write().await.remove(pool_key);
self.postgres_cancel_contexts.write().await.remove(pool_key);
let removed = self.connections.write().await.remove(pool_key);
if let Some(pool) = removed {
close_removed_pools_in_background(&self.task_supervisor, vec![(pool_key.to_string(), pool)]);
true
} else {
false
}
}
#[cfg(feature = "duckdb-bundled")]
async fn remove_pool_if_duckdb_isolation_mismatch(&self, pool_key: &str) -> bool {
let isolation_enabled = self.duckdb_worker_process_isolation.load(Ordering::Relaxed);
let mismatch = {
let connections = self.connections.read().await;
match connections.get(pool_key) {
Some(PoolKind::DuckDb(_)) => isolation_enabled,
Some(PoolKind::DuckDbWorker(_)) => !isolation_enabled,
_ => false,
}
};
if mismatch {
self.remove_pool_by_key_detached(pool_key).await
} else {
false
}
}
#[cfg(not(feature = "duckdb-bundled"))]
async fn remove_pool_if_duckdb_isolation_mismatch(&self, _pool_key: &str) -> bool { async fn remove_pool_if_duckdb_isolation_mismatch(&self, _pool_key: &str) -> bool {
false false
} }
#[cfg(feature = "duckdb-bundled")]
pub fn spawn_duckdb_pool_cleanup(&self, pool_key: String, con: DuckDbHandle) {
let connections = self.connections.clone();
let supervisor = self.task_supervisor.clone();
let pool_activity = self.pool_activity.clone();
let postgres_cancel_contexts = self.postgres_cancel_contexts.clone();
let task_key = format!("duckdb-cleanup:{pool_key}");
supervisor.clone().spawn_replace(task_key, move |shutdown| async move {
while Arc::strong_count(&con) > 2 {
tokio::select! {
_ = shutdown.cancelled() => return,
_ = tokio::time::sleep(Duration::from_millis(50)) => {}
}
}
supervisor.stop(&format!("keepalive:{pool_key}"));
pool_activity.write().await.remove(&pool_key);
postgres_cancel_contexts.write().await.remove(&pool_key);
let removed = {
let mut conns = connections.write().await;
match conns.get(&pool_key) {
Some(PoolKind::DuckDb(current)) if Arc::ptr_eq(current, &con) => conns.remove(&pool_key),
_ => None,
}
};
if let Some(pool) = removed {
// Keep the old DuckDB pool marked as draining until it is no longer
// visible in the pool map, otherwise a concurrent query could reuse it.
con.clear_draining();
drop(con);
close_pool_kind_with_timeout(pool_key, pool).await;
}
});
}
#[cfg(feature = "duckdb-bundled")]
pub fn spawn_duckdb_draining_cleanup(
&self,
pool_key: String,
con: DuckDbHandle,
mut task: JoinHandle<Result<db::QueryResult, String>>,
) {
let connections = self.connections.clone();
let supervisor = self.task_supervisor.clone();
let pool_activity = self.pool_activity.clone();
let postgres_cancel_contexts = self.postgres_cancel_contexts.clone();
let task_key = format!("duckdb-draining:{pool_key}");
supervisor.clone().spawn_replace(task_key, move |shutdown| async move {
tokio::select! {
_ = shutdown.cancelled() => {
task.abort();
return;
}
_ = &mut task => {}
}
while Arc::strong_count(&con) > 2 {
tokio::select! {
_ = shutdown.cancelled() => return,
_ = tokio::time::sleep(Duration::from_millis(50)) => {}
}
}
supervisor.stop(&format!("keepalive:{pool_key}"));
pool_activity.write().await.remove(&pool_key);
postgres_cancel_contexts.write().await.remove(&pool_key);
let removed = {
let mut conns = connections.write().await;
match conns.get(&pool_key) {
Some(PoolKind::DuckDb(current)) if Arc::ptr_eq(current, &con) => conns.remove(&pool_key),
_ => None,
}
};
if let Some(pool) = removed {
// Keep the old DuckDB pool marked as draining until it is no longer
// visible in the pool map, otherwise a concurrent query could reuse it.
con.clear_draining();
drop(con);
close_pool_kind_with_timeout(pool_key, pool).await;
}
});
}
pub async fn close_database_pool(&self, connection_id: &str, database: Option<&str>) -> Result<bool, String> { pub async fn close_database_pool(&self, connection_id: &str, database: Option<&str>) -> Result<bool, String> {
let db_type = { let db_type = {
let configs = self.configs.read().await; let configs = self.configs.read().await;
@@ -2732,6 +2611,11 @@ impl AppState {
let mut keys = HashSet::new(); let mut keys = HashSet::new();
for (pool_key, pool) in connections.iter() { for (pool_key, pool) in connections.iter() {
#[cfg(feature = "duckdb-sidecar")]
if matches!(pool, PoolKind::DuckDbWorker(_)) {
keys.insert("duckdb".to_string());
continue;
}
if !matches!(pool, PoolKind::Agent(_)) { if !matches!(pool, PoolKind::Agent(_)) {
continue; continue;
} }
@@ -3092,9 +2976,7 @@ impl AppState {
} }
} }
PoolKind::Sqlite(_) PoolKind::Sqlite(_)
| PoolKind::DuckDb(_)
| PoolKind::DuckDbWorker(_) | PoolKind::DuckDbWorker(_)
| PoolKind::ExternalTabular(_)
| PoolKind::ExternalDriver { .. } | PoolKind::ExternalDriver { .. }
| PoolKind::MessageQueue | PoolKind::MessageQueue
| PoolKind::Nacos => true, | PoolKind::Nacos => true,
@@ -3162,13 +3044,13 @@ impl AppState {
close_removed_pools_in_background(&self.task_supervisor, removed); close_removed_pools_in_background(&self.task_supervisor, removed);
} }
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
async fn remove_duckdb_pools_detached(&self) { async fn remove_duckdb_pools_detached(&self) {
let removed = self.drain_duckdb_pools().await; let removed = self.drain_duckdb_pools().await;
close_removed_pools_in_background(&self.task_supervisor, removed); close_removed_pools_in_background(&self.task_supervisor, removed);
} }
#[cfg(not(feature = "duckdb-bundled"))] #[cfg(not(feature = "duckdb-sidecar"))]
async fn remove_duckdb_pools_detached(&self) {} async fn remove_duckdb_pools_detached(&self) {}
pub async fn remove_external_driver_pools(&self, driver_id: &str) { pub async fn remove_external_driver_pools(&self, driver_id: &str) {
@@ -3209,16 +3091,16 @@ impl AppState {
removed removed
} }
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
async fn drain_duckdb_pools(&self) -> Vec<(String, PoolKind)> { async fn drain_duckdb_pools(&self) -> Vec<(String, PoolKind)> {
let keys_to_remove: Vec<String> = self let keys_to_remove: Vec<String> = self
.connections .connections
.read() .read()
.await .await
.iter() .iter()
.filter_map(|(key, pool)| match pool { .filter_map(|(key, pool)| {
PoolKind::DuckDb(_) | PoolKind::DuckDbWorker(_) => Some(key.clone()), let is_duckdb = matches!(pool, PoolKind::DuckDbWorker(_));
_ => None, is_duckdb.then(|| key.clone())
}) })
.collect(); .collect();
self.stop_keepalive_tasks(&keys_to_remove).await; self.stop_keepalive_tasks(&keys_to_remove).await;
@@ -3516,13 +3398,9 @@ fn clone_pool_kind(pool: &PoolKind) -> PoolKind {
PoolKind::Rqlite(client) => PoolKind::Rqlite(client.clone()), PoolKind::Rqlite(client) => PoolKind::Rqlite(client.clone()),
PoolKind::Turso(client) => PoolKind::Turso(client.clone()), PoolKind::Turso(client) => PoolKind::Turso(client.clone()),
PoolKind::CloudflareD1(client) => PoolKind::CloudflareD1(client.clone()), PoolKind::CloudflareD1(client) => PoolKind::CloudflareD1(client.clone()),
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
PoolKind::DuckDb(con) => PoolKind::DuckDb(con.clone()),
#[cfg(feature = "duckdb-bundled")]
PoolKind::DuckDbWorker(client) => PoolKind::DuckDbWorker(client.clone()), PoolKind::DuckDbWorker(client) => PoolKind::DuckDbWorker(client.clone()),
#[cfg(not(feature = "duckdb-bundled"))] #[cfg(not(feature = "duckdb-sidecar"))]
PoolKind::DuckDb(_) => PoolKind::DuckDb(()),
#[cfg(not(feature = "duckdb-bundled"))]
PoolKind::DuckDbWorker(_) => PoolKind::DuckDbWorker(()), PoolKind::DuckDbWorker(_) => PoolKind::DuckDbWorker(()),
PoolKind::MongoDb(client) => PoolKind::MongoDb(client.clone()), PoolKind::MongoDb(client) => PoolKind::MongoDb(client.clone()),
PoolKind::ClickHouse(client) => PoolKind::ClickHouse(client.clone()), PoolKind::ClickHouse(client) => PoolKind::ClickHouse(client.clone()),
@@ -3532,10 +3410,6 @@ fn clone_pool_kind(pool: &PoolKind) -> PoolKind {
PoolKind::VectorDb(client) => PoolKind::VectorDb(client.clone()), PoolKind::VectorDb(client) => PoolKind::VectorDb(client.clone()),
PoolKind::InfluxDb(client) => PoolKind::InfluxDb(client.clone()), PoolKind::InfluxDb(client) => PoolKind::InfluxDb(client.clone()),
PoolKind::Agent(client) => PoolKind::Agent(client.clone()), PoolKind::Agent(client) => PoolKind::Agent(client.clone()),
#[cfg(feature = "duckdb-bundled")]
PoolKind::ExternalTabular(ext) => PoolKind::ExternalTabular(ext.clone()),
#[cfg(not(feature = "duckdb-bundled"))]
PoolKind::ExternalTabular(_) => PoolKind::ExternalTabular(()),
PoolKind::ExternalDriver { driver_id, config, session } => { PoolKind::ExternalDriver { driver_id, config, session } => {
PoolKind::ExternalDriver { driver_id: driver_id.clone(), config: config.clone(), session: session.clone() } PoolKind::ExternalDriver { driver_id: driver_id.clone(), config: config.clone(), session: session.clone() }
} }
@@ -3558,17 +3432,11 @@ pub async fn close_pool_kind(pool: PoolKind) {
PoolKind::Redis(conn) => { PoolKind::Redis(conn) => {
drop(conn); drop(conn);
} }
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
PoolKind::DuckDb(con) => {
crate::db::duckdb_driver::close_connection(con);
}
#[cfg(feature = "duckdb-bundled")]
PoolKind::DuckDbWorker(client) => { PoolKind::DuckDbWorker(client) => {
client.shutdown().await; client.shutdown().await;
} }
#[cfg(not(feature = "duckdb-bundled"))] #[cfg(not(feature = "duckdb-sidecar"))]
PoolKind::DuckDb(_) => {}
#[cfg(not(feature = "duckdb-bundled"))]
PoolKind::DuckDbWorker(_) => {} PoolKind::DuckDbWorker(_) => {}
PoolKind::MongoDb(client) => { PoolKind::MongoDb(client) => {
drop(client); drop(client);
@@ -3595,7 +3463,6 @@ pub async fn close_pool_kind(pool: PoolKind) {
let mut client = client.lock().await; let mut client = client.lock().await;
let _ = client.disconnect().await; let _ = client.disconnect().await;
} }
PoolKind::ExternalTabular(_) => {}
PoolKind::ExternalDriver { session, .. } => { PoolKind::ExternalDriver { session, .. } => {
session.shutdown().await; session.shutdown().await;
} }
@@ -5108,51 +4975,6 @@ mod tests {
let _ = std::fs::remove_dir_all(dir); let _ = std::fs::remove_dir_all(dir);
} }
#[cfg(feature = "duckdb-bundled")]
#[tokio::test]
async fn duckdb_connection_test_does_not_reuse_stale_pool_config() {
let (state, dir) = test_app_state().await;
let db_path = dir.join("app.duckdb");
duckdb::Connection::open(&db_path).unwrap();
let mut config = mysql_config(None);
config.id = "duckdb-conn".to_string();
config.name = "DuckDB".to_string();
config.db_type = DatabaseType::DuckDb;
config.host = db_path.to_string_lossy().to_string();
config.port = 0;
state.configs.write().await.insert(config.id.clone(), config.clone());
state.get_or_create_pool("duckdb-conn", None).await.unwrap();
config.init_script = Some("SELECT definitely_invalid_syntax(".to_string());
let error = state.test_duckdb_connection_config(&config).await.expect_err("invalid submitted script must fail");
assert!(error.contains("Connection init script statement 1 failed"));
let _ = std::fs::remove_dir_all(dir);
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test]
async fn duckdb_connection_test_validates_attached_databases() {
let (state, dir) = test_app_state().await;
let mut config = mysql_config(None);
config.id = "duckdb-conn".to_string();
config.name = "DuckDB".to_string();
config.db_type = DatabaseType::DuckDb;
config.host = ":memory:".to_string();
config.port = 0;
config.attached_databases.push(AttachedDatabaseConfig {
name: "missing".to_string(),
path: dir.join("missing-parent").join("missing.duckdb").to_string_lossy().to_string(),
});
let error = state.test_duckdb_connection_config(&config).await.expect_err("invalid attach must fail");
assert!(error.to_ascii_lowercase().contains("attach"));
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test] #[tokio::test]
async fn sqlite_connection_restores_attached_databases() { async fn sqlite_connection_restores_attached_databases() {
let (state, dir) = test_app_state().await; let (state, dir) = test_app_state().await;
@@ -5189,72 +5011,6 @@ mod tests {
let _ = std::fs::remove_dir_all(dir); let _ = std::fs::remove_dir_all(dir);
} }
#[cfg(feature = "duckdb-bundled")]
#[tokio::test]
async fn duckdb_connection_test_supports_memory_bootstrap() {
let (state, dir) = test_app_state().await;
let mut config = mysql_config(None);
config.id = "duckdb-memory".to_string();
config.name = "DuckDB memory".to_string();
config.db_type = DatabaseType::DuckDb;
config.host = ":memory:".to_string();
config.port = 0;
config.init_script = Some(r#"CREATE TABLE probe AS SELECT E'it\'s;ok' AS value;"#.to_string());
state.test_duckdb_connection_config(&config).await.expect("memory bootstrap succeeds");
let _ = std::fs::remove_dir_all(dir);
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test]
async fn applying_duckdb_worker_isolation_drops_existing_duckdb_pools() {
let (state, dir) = test_app_state().await;
let db_path = dir.join("app.duckdb");
duckdb::Connection::open(&db_path).unwrap();
let mut config = mysql_config(None);
config.id = "duckdb-conn".to_string();
config.name = "DuckDB".to_string();
config.db_type = DatabaseType::DuckDb;
config.host = db_path.to_string_lossy().to_string();
config.port = 0;
state.configs.write().await.insert(config.id.clone(), config);
state.get_or_create_pool("duckdb-conn", None).await.unwrap();
assert!(matches!(state.connections.read().await.get("duckdb-conn"), Some(PoolKind::DuckDb(_))));
state.apply_duckdb_worker_process_isolation(true).await;
assert!(!state.connections.read().await.contains_key("duckdb-conn"));
let _ = std::fs::remove_dir_all(dir);
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test]
async fn duckdb_pool_mode_mismatch_removes_existing_pool() {
let (state, dir) = test_app_state().await;
let db_path = dir.join("app.duckdb");
duckdb::Connection::open(&db_path).unwrap();
let mut config = mysql_config(None);
config.id = "duckdb-conn".to_string();
config.name = "DuckDB".to_string();
config.db_type = DatabaseType::DuckDb;
config.host = db_path.to_string_lossy().to_string();
config.port = 0;
state.configs.write().await.insert(config.id.clone(), config);
state.get_or_create_pool("duckdb-conn", None).await.unwrap();
assert!(matches!(state.connections.read().await.get("duckdb-conn"), Some(PoolKind::DuckDb(_))));
state.set_duckdb_worker_process_isolation_enabled(true);
assert!(state.remove_pool_if_duckdb_isolation_mismatch("duckdb-conn").await);
assert!(!state.connections.read().await.contains_key("duckdb-conn"));
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test] #[tokio::test]
async fn remove_connection_pools_clears_base_and_database_scoped_pools() { async fn remove_connection_pools_clears_base_and_database_scoped_pools() {
let (state, dir) = test_app_state().await; let (state, dir) = test_app_state().await;
@@ -5519,33 +5275,6 @@ for line in sys.stdin:
assert!(!super::is_agent_validate_connection_unsupported("Agent RPC error (-1): Unknown method: kv_put")); assert!(!super::is_agent_validate_connection_unsupported("Agent RPC error (-1): Unknown method: kv_put"));
} }
#[cfg(feature = "duckdb-bundled")]
#[tokio::test]
async fn duckdb_client_session_reuses_base_pool_to_avoid_file_locks() {
let (state, dir) = test_app_state().await;
let db_path = dir.join("session.duckdb");
duckdb::Connection::open(&db_path).unwrap();
let mut config = mysql_config(None);
config.id = "duckdb-conn".to_string();
config.name = "DuckDB".to_string();
config.db_type = DatabaseType::DuckDb;
config.host = db_path.to_string_lossy().to_string();
config.port = 0;
state.configs.write().await.insert(config.id.clone(), config.clone());
let base_pool_key = state.get_or_create_pool("duckdb-conn", None).await.unwrap();
let pool_key = state.get_or_create_pool_for_session("duckdb-conn", Some("main"), Some("tab-1")).await.unwrap();
assert_eq!(pool_key, base_pool_key);
assert!(!state.close_client_session_pool("duckdb-conn", Some("main"), "tab-1").await.unwrap());
let conns = state.connections.read().await;
assert!(conns.contains_key("duckdb-conn"));
assert!(!conns.contains_key("duckdb-conn:session:tab-1"));
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test] #[tokio::test]
async fn sqlite_memory_client_sessions_share_the_same_database() { async fn sqlite_memory_client_sessions_share_the_same_database() {
let (state, dir) = test_app_state().await; let (state, dir) = test_app_state().await;
+3 -3
View File
@@ -1,7 +1,7 @@
//! Quote-aware helpers for inspecting DuckDB SQL snippets without a full //! Quote-aware helpers for inspecting DuckDB SQL snippets without a full
//! parser. Shared by the DuckDB worker (execute-time ATTACH tracking) and the //! parser. Shared by the DuckDB worker (execute-time ATTACH tracking) and the
//! connection layer (init-script handling), and kept free of `duckdb-bundled` //! connection layer (init-script handling). It stays independent of the native
//! so config-derived metadata works in every build. //! DuckDB engine so config-derived metadata remains in the main application.
use crate::models::connection::ConnectionConfig; use crate::models::connection::ConnectionConfig;
@@ -130,7 +130,7 @@ pub fn attached_name_from_attach_sql(sql: &str) -> Option<String> {
parse_identifier_after_as(&trimmed[as_index + 2..]) parse_identifier_after_as(&trimmed[as_index + 2..])
} }
pub(crate) fn strip_leading_comments(sql: &str) -> &str { pub fn strip_leading_comments(sql: &str) -> &str {
let mut rest = sql.trim_start(); let mut rest = sql.trim_start();
loop { loop {
if let Some(after) = rest.strip_prefix("--") { if let Some(after) = rest.strip_prefix("--") {
+89 -13
View File
@@ -1,7 +1,8 @@
#![cfg(feature = "duckdb-bundled")] #![cfg(feature = "duckdb-sidecar")]
use std::collections::HashMap; use std::collections::HashMap;
use std::path::PathBuf; use std::ffi::OsString;
use std::path::{Path, PathBuf};
use std::process::Stdio; use std::process::Stdio;
use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex as StdMutex, OnceLock}; use std::sync::{Arc, Mutex as StdMutex, OnceLock};
@@ -14,20 +15,21 @@ use tokio_util::sync::CancellationToken;
use crate::db; use crate::db;
use crate::db::duckdb_worker_protocol::{ use crate::db::duckdb_worker_protocol::{
DuckDbWorkerConnectParams, DuckDbWorkerError, DuckDbWorkerExecuteParams, DuckDbWorkerMethod, DuckDbWorkerColumnParams, DuckDbWorkerConnectParams, DuckDbWorkerError, DuckDbWorkerExecuteParams,
DuckDbWorkerObjectSourceParams, DuckDbWorkerRequest, DuckDbWorkerResponse, DuckDbWorkerMethod, DuckDbWorkerObjectSourceParams, DuckDbWorkerRequest, DuckDbWorkerResponse,
}; };
use crate::models::connection::AttachedDatabaseConfig; use crate::models::connection::AttachedDatabaseConfig;
use crate::storage::{normalize_duckdb_worker_max_processes, DUCKDB_WORKER_MAX_PROCESSES_DEFAULT}; use crate::storage::{normalize_duckdb_worker_max_processes, DUCKDB_WORKER_MAX_PROCESSES_DEFAULT};
/// Error code the worker reports when a query error left the DuckDB connection poisoned /// Error code the worker reports when a query error left the DuckDB connection poisoned
/// (see the duckdb-rs Parser Error bug). The client kills the worker on this code so the /// (see the duckdb-rs Parser Error bug). The client kills the worker on this code so the
/// next request starts a fresh one. Must match the code emitted in `duckdb_worker_runtime`. /// next request starts a fresh one. Must match the code emitted by the standalone driver runtime.
const DUCKDB_WORKER_POISONED_CODE: &str = "duckdb_worker_poisoned"; const DUCKDB_WORKER_POISONED_CODE: &str = "duckdb_worker_poisoned";
const DUCKDB_WORKER_REQUEST_TIMEOUT_CODE: &str = "duckdb_worker_request_timeout"; const DUCKDB_WORKER_REQUEST_TIMEOUT_CODE: &str = "duckdb_worker_request_timeout";
const DEFAULT_WORKER_REQUEST_TIMEOUT: Duration = Duration::from_secs(60); const DEFAULT_WORKER_REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
const DEFAULT_WORKER_KILL_WAIT: Duration = Duration::from_secs(3); const DEFAULT_WORKER_KILL_WAIT: Duration = Duration::from_secs(3);
const DEFAULT_WORKER_START_WAIT: Duration = Duration::from_secs(5); const DEFAULT_WORKER_START_WAIT: Duration = Duration::from_secs(5);
pub const DUCKDB_DRIVER_PATH_ENV: &str = "DBX_DUCKDB_DRIVER_PATH";
type PendingRequests = Arc<Mutex<HashMap<String, PendingRequest>>>; type PendingRequests = Arc<Mutex<HashMap<String, PendingRequest>>>;
struct PendingRequest { struct PendingRequest {
@@ -48,6 +50,7 @@ struct DuckDbWorkerClientInner {
process_limiter: Arc<Semaphore>, process_limiter: Arc<Semaphore>,
process_limit: usize, process_limit: usize,
executable: PathBuf, executable: PathBuf,
executable_args: Vec<OsString>,
connect_params: DuckDbWorkerConnectParams, connect_params: DuckDbWorkerConnectParams,
request_timeout: Duration, request_timeout: Duration,
worker_start_timeout: Duration, worker_start_timeout: Duration,
@@ -68,8 +71,16 @@ impl DuckDbWorkerClient {
attached_databases: Vec<AttachedDatabaseConfig>, attached_databases: Vec<AttachedDatabaseConfig>,
init_script: Option<String>, init_script: Option<String>,
) -> Result<Self, String> { ) -> Result<Self, String> {
let executable = std::env::current_exe().map_err(|e| e.to_string())?; let (executable, executable_args) = resolve_duckdb_driver_command()?;
Self::open_with_executable(executable, path, attached_databases, init_script).await Self::open_with_command_and_process_limit(
executable,
executable_args,
path,
attached_databases,
init_script,
DUCKDB_WORKER_MAX_PROCESSES_DEFAULT,
)
.await
} }
pub async fn open_with_process_limit( pub async fn open_with_process_limit(
@@ -78,9 +89,16 @@ impl DuckDbWorkerClient {
init_script: Option<String>, init_script: Option<String>,
process_limit: usize, process_limit: usize,
) -> Result<Self, String> { ) -> Result<Self, String> {
let executable = std::env::current_exe().map_err(|e| e.to_string())?; let (executable, executable_args) = resolve_duckdb_driver_command()?;
Self::open_with_executable_and_process_limit(executable, path, attached_databases, init_script, process_limit) Self::open_with_command_and_process_limit(
.await executable,
executable_args,
path,
attached_databases,
init_script,
process_limit,
)
.await
} }
pub async fn open_with_executable( pub async fn open_with_executable(
@@ -105,9 +123,29 @@ impl DuckDbWorkerClient {
attached_databases: Vec<AttachedDatabaseConfig>, attached_databases: Vec<AttachedDatabaseConfig>,
init_script: Option<String>, init_script: Option<String>,
process_limit: usize, process_limit: usize,
) -> Result<Self, String> {
Self::open_with_command_and_process_limit(
executable,
Vec::new(),
path,
attached_databases,
init_script,
process_limit,
)
.await
}
async fn open_with_command_and_process_limit(
executable: PathBuf,
executable_args: Vec<OsString>,
path: String,
attached_databases: Vec<AttachedDatabaseConfig>,
init_script: Option<String>,
process_limit: usize,
) -> Result<Self, String> { ) -> Result<Self, String> {
let client = Self::new_unconnected_with_timeouts( let client = Self::new_unconnected_with_timeouts(
executable, executable,
executable_args,
path, path,
attached_databases, attached_databases,
init_script, init_script,
@@ -122,6 +160,7 @@ impl DuckDbWorkerClient {
#[doc(hidden)] #[doc(hidden)]
pub fn new_unconnected_with_timeouts( pub fn new_unconnected_with_timeouts(
executable: PathBuf, executable: PathBuf,
executable_args: Vec<OsString>,
path: String, path: String,
attached_databases: Vec<AttachedDatabaseConfig>, attached_databases: Vec<AttachedDatabaseConfig>,
init_script: Option<String>, init_script: Option<String>,
@@ -139,6 +178,7 @@ impl DuckDbWorkerClient {
process_limiter, process_limiter,
process_limit, process_limit,
executable, executable,
executable_args,
connect_params: DuckDbWorkerConnectParams { path, attached_databases, init_script }, connect_params: DuckDbWorkerConnectParams { path, attached_databases, init_script },
request_timeout, request_timeout,
worker_start_timeout, worker_start_timeout,
@@ -246,6 +286,18 @@ impl DuckDbWorkerClient {
.await .await
} }
pub async fn get_table_ddl(&self, database: String, schema: String, table: String) -> Result<String, String> {
self.metadata_request(DuckDbWorkerMethod::GetTableDdl, DuckDbWorkerColumnParams { database, schema, table })
.await
}
pub async fn completion_assistant(
&self,
request: db::CompletionAssistantRequest,
) -> Result<db::CompletionAssistantResponse, String> {
self.metadata_request(DuckDbWorkerMethod::CompletionAssistant, request).await
}
pub async fn get_object_source( pub async fn get_object_source(
&self, &self,
database: String, database: String,
@@ -413,9 +465,14 @@ impl DuckDbWorkerClient {
.map_err(|_| duckdb_worker_process_limit_error(self.inner.process_limit))? .map_err(|_| duckdb_worker_process_limit_error(self.inner.process_limit))?
.map_err(|_| "DuckDB worker process limiter is closed".to_string())?; .map_err(|_| "DuckDB worker process limiter is closed".to_string())?;
log::info!("[duckdb-worker:start] executable={}", self.inner.executable.display()); log::info!(
let mut child = crate::process::new_tokio_command(&self.inner.executable) "[duckdb-worker:start] executable={} args={:?}",
.arg("--duckdb-worker") self.inner.executable.display(),
self.inner.executable_args
);
let mut command = crate::process::new_tokio_command(&self.inner.executable);
command.args(&self.inner.executable_args);
let mut child = command
.stdin(Stdio::piped()) .stdin(Stdio::piped())
.stdout(Stdio::piped()) .stdout(Stdio::piped())
.stderr(Stdio::inherit()) .stderr(Stdio::inherit())
@@ -524,6 +581,25 @@ impl DuckDbWorkerClient {
} }
} }
fn resolve_duckdb_driver_command() -> Result<(PathBuf, Vec<OsString>), String> {
if let Some(executable) = std::env::var_os(DUCKDB_DRIVER_PATH_ENV).filter(|value| !value.is_empty()) {
let executable = PathBuf::from(executable);
ensure_duckdb_driver_exists(&executable)?;
return Ok((executable, Vec::new()));
}
Err(format!(
"DuckDB driver is not installed. Please install it from the Driver Manager or set {DUCKDB_DRIVER_PATH_ENV}."
))
}
fn ensure_duckdb_driver_exists(executable: &Path) -> Result<(), String> {
if executable.is_file() {
return Ok(());
}
Err(format!("DuckDB driver configured by {DUCKDB_DRIVER_PATH_ENV} was not found: {}", executable.display()))
}
struct WorkerProcessLimiterState { struct WorkerProcessLimiterState {
limit: usize, limit: usize,
limiter: Arc<Semaphore>, limiter: Arc<Semaphore>,
@@ -1,4 +1,4 @@
#![cfg(feature = "duckdb-bundled")] #![cfg(feature = "duckdb-sidecar")]
use serde::{de::DeserializeOwned, Deserialize, Serialize}; use serde::{de::DeserializeOwned, Deserialize, Serialize};
@@ -35,6 +35,8 @@ pub enum DuckDbWorkerMethod {
ListSchemas, ListSchemas,
ListTables, ListTables,
ListColumns, ListColumns,
GetTableDdl,
CompletionAssistant,
GetObjectSource, GetObjectSource,
AttachDatabase, AttachDatabase,
Cancel, Cancel,
+2 -5
View File
@@ -5,14 +5,11 @@ pub mod cloudflare_d1;
pub use cloudflare_d1 as cloudflare_d1_driver; pub use cloudflare_d1 as cloudflare_d1_driver;
pub mod document_result; pub mod document_result;
pub mod doris; pub mod doris;
pub mod duckdb_driver;
pub mod duckdb_sql; pub mod duckdb_sql;
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
pub mod duckdb_worker_process; pub mod duckdb_worker_process;
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
pub mod duckdb_worker_protocol; pub mod duckdb_worker_protocol;
#[cfg(feature = "duckdb-bundled")]
pub mod duckdb_worker_runtime;
pub mod elasticsearch_driver; pub mod elasticsearch_driver;
pub mod elasticsearch_sql; pub mod elasticsearch_sql;
pub mod file_validator; pub mod file_validator;
+3 -3
View File
@@ -63,7 +63,7 @@ pub enum DatabaseCreationTarget {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
pub struct DuckDbAttachDatabaseSqlOptions { pub struct DuckDbAttachDatabaseSqlOptions {
pub path: String, pub path: String,
pub name: String, pub name: String,
@@ -312,7 +312,7 @@ pub fn supports_database_property_comment(database_type: Option<DatabaseType>) -
) )
} }
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
pub fn build_duckdb_attach_database_sql(options: DuckDbAttachDatabaseSqlOptions) -> String { pub fn build_duckdb_attach_database_sql(options: DuckDbAttachDatabaseSqlOptions) -> String {
format!( format!(
"ATTACH {} AS {};", "ATTACH {} AS {};",
@@ -1096,7 +1096,7 @@ mod tests {
assert!(!supports_create_database_charset(Some(DatabaseType::Postgres), None)); assert!(!supports_create_database_charset(Some(DatabaseType::Postgres), None));
} }
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
#[test] #[test]
fn builds_duckdb_attach_sql() { fn builds_duckdb_attach_sql() {
assert_eq!( assert_eq!(
-173
View File
@@ -1,173 +0,0 @@
#![cfg(feature = "duckdb-bundled")]
use std::sync::Arc;
use super::types::{CacheState, ExternalTableRef, ExternalTableSnapshot};
/// Loads an external table snapshot into an in-memory DuckDB connection.
pub fn load_snapshot_to_duckdb(con: &duckdb::Connection, snapshot: &ExternalTableSnapshot) -> Result<(), String> {
let table_name = sanitize_table_name(&snapshot.table_ref.table_name);
con.execute(&format!("DROP TABLE IF EXISTS \"{}\"", table_name), [])
.map_err(|e| format!("Failed to drop table: {e}"))?;
let col_defs: Vec<String> = snapshot
.columns
.iter()
.map(|c| {
format!(
"\"{}\" {}{}",
c.name.replace('"', "\"\""),
c.duckdb_type,
if c.is_nullable { "" } else { " NOT NULL" }
)
})
.collect();
let pk_cols: Vec<&str> = snapshot.columns.iter().filter(|c| c.is_primary_key).map(|c| c.name.as_str()).collect();
let mut create_sql = format!("CREATE TABLE \"{}\" ({}", table_name, col_defs.join(", "));
if !pk_cols.is_empty() {
create_sql.push_str(&format!(
", PRIMARY KEY ({})",
pk_cols.iter().map(|c| format!("\"{}\"", c.replace('"', "\"\""))).collect::<Vec<_>>().join(", ")
));
}
create_sql.push(')');
con.execute(&create_sql, []).map_err(|e| format!("Failed to create table: {e}"))?;
if snapshot.rows.is_empty() {
return Ok(());
}
let placeholders: Vec<&str> = (0..snapshot.columns.len()).map(|_| "?").collect();
let insert_sql = format!("INSERT INTO \"{}\" VALUES ({})", table_name, placeholders.join(", "));
let mut stmt = con.prepare(&insert_sql).map_err(|e| format!("Failed to prepare insert: {e}"))?;
for row in &snapshot.rows {
let params: Vec<Box<dyn duckdb::ToSql>> = row
.iter()
.enumerate()
.map(|(i, val)| {
let col_type = snapshot.columns.get(i).map(|c| c.duckdb_type.as_str()).unwrap_or("VARCHAR");
json_to_duckdb_param(val, col_type)
})
.collect();
let param_refs: Vec<&dyn duckdb::ToSql> = params.iter().map(|p| p.as_ref()).collect();
stmt.execute(param_refs.as_slice()).map_err(|e| format!("Failed to insert row: {e}"))?;
}
Ok(())
}
fn json_to_duckdb_param(val: &serde_json::Value, col_type: &str) -> Box<dyn duckdb::ToSql> {
match val {
serde_json::Value::Null => Box::new(None::<String>),
serde_json::Value::Bool(b) => Box::new(*b),
serde_json::Value::Number(n) => {
if col_type.contains("INT") {
Box::new(n.as_i64().unwrap_or(0))
} else if col_type.contains("DOUBLE") || col_type.contains("FLOAT") || col_type.contains("DECIMAL") {
Box::new(n.as_f64().unwrap_or(0.0))
} else {
Box::new(n.to_string())
}
}
serde_json::Value::String(s) => Box::new(s.clone()),
_ => Box::new(val.to_string()),
}
}
/// Sanitize a source table name into a stable DuckDB table identifier.
pub fn sanitize_table_name(name: &str) -> String {
let sanitized: String = name.chars().map(|c| if c.is_alphanumeric() || c == '_' { c } else { '_' }).collect();
if sanitized.is_empty() {
"_unnamed_".to_string()
} else if sanitized.chars().next().is_some_and(|c| c.is_ascii_digit()) {
format!("_{sanitized}")
} else {
sanitized
}
}
/// Manages a DuckDB cache for one external source.
pub struct ExternalPool {
pub source: Arc<dyn super::traits::ExternalTabularSource>,
pub cache: Arc<std::sync::Mutex<duckdb::Connection>>,
pub cache_state: std::sync::Mutex<CacheState>,
pub table_map: std::sync::Mutex<std::collections::HashMap<String, ExternalTableRef>>,
}
impl ExternalPool {
pub fn new(
source: Arc<dyn super::traits::ExternalTabularSource>,
cache: Arc<std::sync::Mutex<duckdb::Connection>>,
) -> Self {
Self {
source,
cache,
cache_state: std::sync::Mutex::new(CacheState::Empty),
table_map: std::sync::Mutex::new(std::collections::HashMap::new()),
}
}
pub async fn refresh_cache(&self) -> Result<(), String> {
{
let mut state = self.cache_state.lock().map_err(|e| e.to_string())?;
*state = CacheState::Loading;
}
let tables = self.source.list_tables().await?;
let mut new_table_map = std::collections::HashMap::new();
for table_ref in &tables {
let snapshot = self.source.load_table(table_ref).await?;
let mut sanitized_name = sanitize_table_name(&table_ref.table_name);
if new_table_map.contains_key(&sanitized_name) {
let base = sanitized_name.clone();
let mut suffix = 2;
loop {
sanitized_name = format!("{base}_{suffix}");
if !new_table_map.contains_key(&sanitized_name) {
break;
}
suffix += 1;
}
}
let cache = self.cache.clone();
tokio::task::spawn_blocking(move || {
let con = cache.lock().map_err(|e| e.to_string())?;
load_snapshot_to_duckdb(&con, &snapshot)
})
.await
.map_err(|e| e.to_string())??;
new_table_map.insert(sanitized_name, table_ref.clone());
}
{
let mut map = self.table_map.lock().map_err(|e| e.to_string())?;
*map = new_table_map;
}
{
let mut state = self.cache_state.lock().map_err(|e| e.to_string())?;
*state = CacheState::Fresh;
}
Ok(())
}
}
impl std::fmt::Debug for ExternalPool {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ExternalPool")
.field("source", &self.source.display_name())
.field("cache_state", &self.cache_state)
.finish()
}
}
-3
View File
@@ -1,8 +1,5 @@
pub mod duckdb_cache;
pub mod traits; pub mod traits;
pub mod types; pub mod types;
#[cfg(feature = "duckdb-bundled")]
pub use duckdb_cache::ExternalPool;
pub use traits::ExternalTabularSource; pub use traits::ExternalTabularSource;
pub use types::*; pub use types::*;
+5 -825
View File
@@ -1,7 +1,3 @@
#[cfg(feature = "duckdb-bundled")]
use chrono::{DateTime, Duration as ChronoDuration, NaiveDate, NaiveDateTime, NaiveTime, Utc};
#[cfg(feature = "duckdb-bundled")]
use duckdb::types::{TimeUnit, Value, ValueRef};
use futures::StreamExt; use futures::StreamExt;
use mysql_async::prelude::Queryable; use mysql_async::prelude::Queryable;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@@ -18,10 +14,6 @@ use std::sync::{
Arc, Arc,
}; };
use std::time::Duration; use std::time::Duration;
#[cfg(feature = "duckdb-bundled")]
use tokio::task::JoinHandle;
#[cfg(feature = "duckdb-bundled")]
use tokio::time::sleep;
use tokio::time::timeout; use tokio::time::timeout;
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
@@ -30,8 +22,6 @@ use crate::database_capabilities;
use crate::db; use crate::db;
use crate::models::connection::{ConnectionConfig, DatabaseType}; use crate::models::connection::{ConnectionConfig, DatabaseType};
use crate::query_execution_sql::is_write_sql; use crate::query_execution_sql::is_write_sql;
#[cfg(feature = "duckdb-bundled")]
use crate::sql::starts_with_duckdb_result_sql_keyword;
use crate::sql::{split_sql_batches, split_sql_statements, starts_with_executable_sql_keyword}; use crate::sql::{split_sql_batches, split_sql_statements, starts_with_executable_sql_keyword};
use crate::sql_dialect::{resolve_for_db, CAP_TRANSACTIONAL_DDL}; use crate::sql_dialect::{resolve_for_db, CAP_TRANSACTIONAL_DDL};
use crate::sql_risk::{classify_sql_risk_for_database, SqlRisk}; use crate::sql_risk::{classify_sql_risk_for_database, SqlRisk};
@@ -45,10 +35,6 @@ pub const QUERY_CANCELED: &str = "Query canceled";
const MONGO_SHELL_COMMAND_HINT: &str = "Use MongoDB shell-style commands, for example: db.collection.find({}).limit(100), db.collection.aggregate([]), db.collection.aggregate([], { explain: true }), db.version(), db.collection.countDocuments({}), db.collection.distinct(\"field\"), db.collection.getIndexes(), db.collection.createIndex({...}), or db.collection.insertOne({...})."; const MONGO_SHELL_COMMAND_HINT: &str = "Use MongoDB shell-style commands, for example: db.collection.find({}).limit(100), db.collection.aggregate([]), db.collection.aggregate([], { explain: true }), db.version(), db.collection.countDocuments({}), db.collection.distinct(\"field\"), db.collection.getIndexes(), db.collection.createIndex({...}), or db.collection.insertOne({...}).";
const SQL_OMITTED_ERROR_CONTEXT: &str = const SQL_OMITTED_ERROR_CONTEXT: &str =
"SQL text omitted from user-facing error; enable debug SQL diagnostics for a redacted statement."; "SQL text omitted from user-facing error; enable debug SQL diagnostics for a redacted statement.";
#[cfg(feature = "duckdb-bundled")]
const DUCKDB_INTERRUPT_DRAIN_TIMEOUT: Duration = Duration::from_secs(2);
#[cfg(feature = "duckdb-bundled")]
const DUCKDB_DRAINING_MESSAGE: &str = "The previous DuckDB query is still stopping. Please try again shortly.";
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PoolErrorAction { pub enum PoolErrorAction {
@@ -476,387 +462,6 @@ fn query_result_row_limit(max_rows: Option<usize>) -> usize {
max_rows.unwrap_or(MAX_ROWS).max(1) max_rows.unwrap_or(MAX_ROWS).max(1)
} }
#[cfg(feature = "duckdb-bundled")]
pub fn duckdb_execute(con: &duckdb::Connection, sql: &str) -> Result<db::QueryResult, String> {
duckdb_execute_with_max_rows(con, sql, None)
}
#[cfg(feature = "duckdb-bundled")]
fn duckdb_value_to_json(row: &duckdb::Row<'_>, idx: usize) -> serde_json::Value {
let Ok(value_ref) = row.get_ref(idx) else {
return serde_json::Value::Null;
};
match value_ref {
ValueRef::Null => serde_json::Value::Null,
ValueRef::Boolean(b) => serde_json::Value::Bool(b),
ValueRef::TinyInt(i) => serde_json::Value::Number((i as i64).into()),
ValueRef::SmallInt(i) => serde_json::Value::Number((i as i64).into()),
ValueRef::Int(i) => serde_json::Value::Number((i as i64).into()),
ValueRef::BigInt(i) => serde_json::Value::Number(i.into()),
ValueRef::HugeInt(i) => serde_json::Value::String(i.to_string()),
ValueRef::UTinyInt(i) => serde_json::Value::Number((i as u64).into()),
ValueRef::USmallInt(i) => serde_json::Value::Number((i as u64).into()),
ValueRef::UInt(i) => serde_json::Value::Number((i as u64).into()),
ValueRef::UBigInt(i) => serde_json::Value::Number(i.into()),
ValueRef::Float(f) => {
serde_json::Number::from_f64(f as f64).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null)
}
ValueRef::Double(f) => {
serde_json::Number::from_f64(f).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null)
}
ValueRef::Decimal(d) => serde_json::Value::String(d.to_string()),
ValueRef::Date32(days) => {
duckdb_date32_to_string(days).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
}
ValueRef::Time64(unit, value) => {
duckdb_time64_to_string(unit, value).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
}
ValueRef::Timestamp(unit, value) => {
duckdb_timestamp_to_string(unit, value).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
}
ValueRef::Text(bytes) => std::str::from_utf8(bytes)
.map(|s| serde_json::Value::String(s.to_string()))
.unwrap_or(serde_json::Value::Null),
ValueRef::Blob(bytes) => {
let hex: String = bytes.iter().map(|b| format!("{:02x}", b)).collect();
serde_json::Value::String(format!("\\x{hex}"))
}
ValueRef::Interval { months, days, nanos } => {
serde_json::Value::String(duckdb_interval_to_string(months, days, nanos))
}
ValueRef::List(..)
| ValueRef::Array(..)
| ValueRef::Struct(..)
| ValueRef::Map(..)
| ValueRef::Enum(..)
| ValueRef::Union(..) => duckdb_owned_value_to_json(&value_ref.to_owned()),
_ => duckdb_owned_value_to_json(&value_ref.to_owned()),
}
}
#[cfg(feature = "duckdb-bundled")]
fn duckdb_owned_value_to_json(value: &Value) -> serde_json::Value {
match value {
Value::Null => serde_json::Value::Null,
Value::Boolean(b) => serde_json::Value::Bool(*b),
Value::TinyInt(i) => serde_json::Value::Number((*i as i64).into()),
Value::SmallInt(i) => serde_json::Value::Number((*i as i64).into()),
Value::Int(i) => serde_json::Value::Number((*i as i64).into()),
Value::BigInt(i) => serde_json::Value::Number((*i).into()),
Value::HugeInt(i) => serde_json::Value::String(i.to_string()),
Value::UTinyInt(i) => serde_json::Value::Number((*i as u64).into()),
Value::USmallInt(i) => serde_json::Value::Number((*i as u64).into()),
Value::UInt(i) => serde_json::Value::Number((*i as u64).into()),
Value::UBigInt(i) => serde_json::Value::Number((*i).into()),
Value::Float(f) => {
serde_json::Number::from_f64(*f as f64).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null)
}
Value::Double(f) => {
serde_json::Number::from_f64(*f).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null)
}
Value::Decimal(d) => serde_json::Value::String(d.to_string()),
Value::Timestamp(unit, value) => {
duckdb_timestamp_to_string(*unit, *value).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
}
Value::Text(text) | Value::Enum(text) => serde_json::Value::String(text.clone()),
Value::Blob(bytes) => {
let hex: String = bytes.iter().map(|b| format!("{:02x}", b)).collect();
serde_json::Value::String(format!("\\x{hex}"))
}
Value::Date32(days) => {
duckdb_date32_to_string(*days).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
}
Value::Time64(unit, value) => {
duckdb_time64_to_string(*unit, *value).map(serde_json::Value::String).unwrap_or(serde_json::Value::Null)
}
Value::Interval { months, days, nanos } => {
serde_json::Value::String(duckdb_interval_to_string(*months, *days, *nanos))
}
Value::List(values) | Value::Array(values) => {
serde_json::Value::Array(values.iter().map(duckdb_owned_value_to_json).collect())
}
Value::Struct(entries) => serde_json::Value::Object(
entries.iter().map(|(key, value)| (key.clone(), duckdb_owned_value_to_json(value))).collect(),
),
Value::Map(entries) => serde_json::Value::Array(
entries
.iter()
.map(|(key, value)| {
serde_json::json!({
"key": duckdb_owned_value_to_json(key),
"value": duckdb_owned_value_to_json(value),
})
})
.collect(),
),
Value::Union(value) => duckdb_owned_value_to_json(value),
_ => serde_json::Value::Null,
}
}
#[cfg(feature = "duckdb-bundled")]
fn duckdb_interval_to_string(months: i32, days: i32, nanos: i64) -> String {
let mut parts = Vec::new();
if months != 0 {
let years = months / 12;
let rem = months % 12;
if years != 0 {
parts.push(format!("{} year{}", years, if years.abs() != 1 { "s" } else { "" }));
}
if rem != 0 {
parts.push(format!("{} mon{}", rem, if rem.abs() != 1 { "s" } else { "" }));
}
}
if days != 0 {
parts.push(format!("{} day{}", days, if days.abs() != 1 { "s" } else { "" }));
}
if nanos != 0 {
let total_secs = nanos / 1_000_000_000;
let hours = total_secs / 3600;
let mins = (total_secs % 3600) / 60;
let secs = total_secs % 60;
let sub_nanos = (nanos % 1_000_000_000).unsigned_abs();
if sub_nanos > 0 {
parts.push(format!(
"{:02}:{:02}:{:02}.{}",
hours,
mins,
secs,
format_temporal_without_empty_fraction(format!("0.{:09}", sub_nanos)).trim_start_matches("0.")
));
} else {
parts.push(format!("{:02}:{:02}:{:02}", hours, mins, secs));
}
}
if parts.is_empty() {
"00:00:00".to_string()
} else {
parts.join(" ")
}
}
#[cfg(feature = "duckdb-bundled")]
fn duckdb_date32_to_string(days: i32) -> Option<String> {
let epoch = NaiveDate::from_ymd_opt(1970, 1, 1)?;
epoch.checked_add_signed(ChronoDuration::days(i64::from(days))).map(|date| date.to_string())
}
#[cfg(feature = "duckdb-bundled")]
fn duckdb_time64_to_string(unit: TimeUnit, value: i64) -> Option<String> {
let nanos = duckdb_time_unit_to_nanos(unit, value)?;
let seconds = nanos.div_euclid(1_000_000_000);
let nanos_remainder = nanos.rem_euclid(1_000_000_000) as u32;
if !(0..86_400).contains(&seconds) {
return None;
}
let time = NaiveTime::from_num_seconds_from_midnight_opt(seconds as u32, nanos_remainder)?;
Some(format_temporal_without_empty_fraction(time.to_string()))
}
#[cfg(feature = "duckdb-bundled")]
fn duckdb_timestamp_to_string(unit: TimeUnit, value: i64) -> Option<String> {
let nanos = duckdb_time_unit_to_nanos(unit, value)?;
let seconds = nanos.div_euclid(1_000_000_000);
let nanos_remainder = nanos.rem_euclid(1_000_000_000) as u32;
let dt: DateTime<Utc> = DateTime::from_timestamp(seconds, nanos_remainder)?;
Some(format_naive_datetime(dt.naive_utc()))
}
#[cfg(feature = "duckdb-bundled")]
fn duckdb_time_unit_to_nanos(unit: TimeUnit, value: i64) -> Option<i64> {
match unit {
TimeUnit::Second => value.checked_mul(1_000_000_000),
TimeUnit::Millisecond => value.checked_mul(1_000_000),
TimeUnit::Microsecond => value.checked_mul(1_000),
TimeUnit::Nanosecond => Some(value),
}
}
#[cfg(feature = "duckdb-bundled")]
fn format_naive_datetime(value: NaiveDateTime) -> String {
if value.and_utc().timestamp_subsec_nanos() == 0 {
value.format("%Y-%m-%d %H:%M:%S").to_string()
} else {
format_temporal_without_empty_fraction(value.to_string())
}
}
#[cfg(feature = "duckdb-bundled")]
fn format_temporal_without_empty_fraction(value: String) -> String {
if !value.contains('.') {
return value;
}
let trimmed = value.trim_end_matches('0').trim_end_matches('.');
trimmed.to_string()
}
#[cfg(feature = "duckdb-bundled")]
pub fn duckdb_execute_with_max_rows(
con: &duckdb::Connection,
sql: &str,
max_rows: Option<usize>,
) -> Result<db::QueryResult, String> {
let start = std::time::Instant::now();
let row_limit = query_result_row_limit(max_rows);
if starts_with_duckdb_result_sql_keyword(sql) {
let mut stmt = con.prepare(sql).map_err(|e| e.to_string())?;
let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
let stmt_ref = rows.as_ref().ok_or("DuckDB statement unavailable")?;
let col_count = stmt_ref.column_count();
let columns: Vec<String> = (0..col_count)
.map(|i| stmt_ref.column_name(i).map(|s| s.to_string()).unwrap_or_else(|_| "?".to_string()))
.collect();
let mut result_rows = Vec::new();
while let Some(row) = rows.next().map_err(|e| e.to_string())? {
let vals: Vec<serde_json::Value> = (0..col_count).map(|i| duckdb_value_to_json(row, i)).collect();
result_rows.push(vals);
if result_rows.len() > row_limit {
break;
}
}
let truncated = result_rows.len() > row_limit;
if truncated {
result_rows.truncate(row_limit);
}
Ok(db::QueryResult {
columns,
column_types: Vec::new(),
column_sortables: vec![],
rows: result_rows,
affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(),
truncated,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
})
} else {
let affected = con.execute(sql, []).map_err(|e| e.to_string())?;
Ok(db::QueryResult {
columns: vec![],
column_types: Vec::new(),
column_sortables: vec![],
rows: vec![],
affected_rows: affected as u64,
execution_time_ms: start.elapsed().as_millis(),
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
})
}
}
#[cfg(feature = "duckdb-bundled")]
enum DuckDbTaskWait {
Finished(Result<db::QueryResult, String>),
Draining { error: String, task: JoinHandle<Result<db::QueryResult, String>> },
}
#[cfg(feature = "duckdb-bundled")]
fn duckdb_join_result(
result: Result<Result<db::QueryResult, String>, tokio::task::JoinError>,
) -> Result<db::QueryResult, String> {
result.map_err(|e| e.to_string())?
}
#[cfg(feature = "duckdb-bundled")]
async fn interrupt_and_drain_duckdb_task(
interrupt_handle: std::sync::Arc<duckdb::InterruptHandle>,
mut task: JoinHandle<Result<db::QueryResult, String>>,
error: String,
) -> DuckDbTaskWait {
interrupt_handle.interrupt();
match timeout(DUCKDB_INTERRUPT_DRAIN_TIMEOUT, &mut task).await {
Ok(result) => {
let _ = result;
DuckDbTaskWait::Finished(Err(error))
}
Err(_) => DuckDbTaskWait::Draining { error, task },
}
}
#[cfg(feature = "duckdb-bundled")]
async fn wait_for_duckdb_task_with_interrupt_outcome(
cancel_token: Option<CancellationToken>,
timeout_duration: Option<Duration>,
interrupt_handle: std::sync::Arc<duckdb::InterruptHandle>,
mut task: JoinHandle<Result<db::QueryResult, String>>,
) -> DuckDbTaskWait {
match (cancel_token, timeout_duration) {
(Some(token), Some(duration)) => {
tokio::select! {
biased;
_ = token.cancelled() => {
interrupt_and_drain_duckdb_task(interrupt_handle, task, canceled_error()).await
}
result = &mut task => DuckDbTaskWait::Finished(duckdb_join_result(result)),
_ = sleep(duration) => {
interrupt_and_drain_duckdb_task(interrupt_handle, task, timeout_error()).await
}
}
}
(Some(token), None) => {
tokio::select! {
biased;
_ = token.cancelled() => {
interrupt_and_drain_duckdb_task(interrupt_handle, task, canceled_error()).await
}
result = &mut task => DuckDbTaskWait::Finished(duckdb_join_result(result)),
}
}
(None, Some(duration)) => {
tokio::select! {
result = &mut task => DuckDbTaskWait::Finished(duckdb_join_result(result)),
_ = sleep(duration) => {
interrupt_and_drain_duckdb_task(interrupt_handle, task, timeout_error()).await
}
}
}
(None, None) => DuckDbTaskWait::Finished(duckdb_join_result(task.await)),
}
}
#[cfg(feature = "duckdb-bundled")]
async fn wait_for_duckdb_task_with_interrupt(
cancel_token: Option<CancellationToken>,
timeout_duration: Option<Duration>,
interrupt_handle: std::sync::Arc<duckdb::InterruptHandle>,
task: JoinHandle<Result<db::QueryResult, String>>,
) -> Result<db::QueryResult, String> {
match wait_for_duckdb_task_with_interrupt_outcome(cancel_token, timeout_duration, interrupt_handle, task).await {
DuckDbTaskWait::Finished(result) => result,
DuckDbTaskWait::Draining { error, .. } => Err(error),
}
}
#[cfg(feature = "duckdb-bundled")]
pub(crate) fn duckdb_execute_for_database(
con: &duckdb::Connection,
attached_names: &[String],
database: Option<&str>,
sql: &str,
max_rows: Option<usize>,
) -> Result<db::QueryResult, String> {
if let Some(database) = database.map(str::trim).filter(|database| !database.is_empty()) {
let catalog = if database == "main" {
crate::schema::duckdb_primary_catalog(con, attached_names)?
} else {
database.to_string()
};
con.execute_batch(&format!("USE {}", duckdb_quote_ident(&catalog))).map_err(|e| e.to_string())?;
}
duckdb_execute_with_max_rows(con, sql, max_rows)
}
#[cfg(feature = "duckdb-bundled")]
fn duckdb_quote_ident(value: &str) -> String {
format!("\"{}\"", value.replace('"', "\"\""))
}
pub fn truncate_result(result: db::QueryResult) -> db::QueryResult { pub fn truncate_result(result: db::QueryResult) -> db::QueryResult {
truncate_result_with_max_rows(result, None) truncate_result_with_max_rows(result, None)
} }
@@ -1166,11 +771,6 @@ where
} }
} }
#[cfg(feature = "duckdb-bundled")]
pub fn duckdb_draining_error() -> String {
DUCKDB_DRAINING_MESSAGE.to_string()
}
pub fn is_canceled(cancel_token: &Option<CancellationToken>) -> bool { pub fn is_canceled(cancel_token: &Option<CancellationToken>) -> bool {
cancel_token.as_ref().map(|token| token.is_cancelled()).unwrap_or(false) cancel_token.as_ref().map(|token| token.is_cancelled()).unwrap_or(false)
} }
@@ -1366,12 +966,10 @@ pub async fn do_execute(
let _activity_touch = state.pool_activity_touch(pool_key); let _activity_touch = state.pool_activity_touch(pool_key);
let query_timeout = resolve_query_timeout(options.timeout_secs); let query_timeout = resolve_query_timeout(options.timeout_secs);
let (_duckdb_attached_names, read_only_connection) = { let read_only_connection = {
let configs = state.configs.read().await; let configs = state.configs.read().await;
let config = crate::connection::config_for_pool_key(pool_key, &configs); let config = crate::connection::config_for_pool_key(pool_key, &configs);
let attached = config.map(crate::db::duckdb_sql::config_attached_names).unwrap_or_default(); config.filter(|config| config.read_only).map(|config| (config.name.clone(), config.db_type))
let connection = config.filter(|config| config.read_only).map(|config| (config.name.clone(), config.db_type));
(attached, connection)
}; };
let operation_budget = operation_budget_for_pool_key(state, pool_key, query_timeout).await; let operation_budget = operation_budget_for_pool_key(state, pool_key, query_timeout).await;
if let Some((name, database_type)) = read_only_connection { if let Some((name, database_type)) = read_only_connection {
@@ -1383,49 +981,7 @@ pub async fn do_execute(
let pool = connections.get(pool_key).ok_or("Connection not found")?; let pool = connections.get(pool_key).ok_or("Connection not found")?;
let result = match pool { let result = match pool {
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
PoolKind::DuckDb(con) => {
let con = con.clone();
if con.is_draining() {
drop(connections);
return Err(duckdb_draining_error());
}
let interrupt_handle = con.interrupt_handle();
if let Some(ref execution_id) = options.execution_id {
let cancel_interrupt_handle = interrupt_handle.clone();
state.running_queries.register_interrupt(execution_id, move || {
cancel_interrupt_handle.interrupt();
});
}
let sql = sql.to_string();
let database = database.map(str::to_string);
let attached_names = _duckdb_attached_names;
let max_rows = options.max_rows;
drop(connections);
let task_con = con.clone();
let task = tokio::task::spawn_blocking(move || {
let con = task_con.lock().map_err(|e| e.to_string())?;
duckdb_execute_for_database(&con, &attached_names, database.as_deref(), &sql, max_rows)
});
let result =
wait_for_duckdb_task_with_interrupt_outcome(cancel_token, query_timeout, interrupt_handle, task).await;
match result {
DuckDbTaskWait::Finished(result) => {
if matches!(result.as_ref(), Err(err) if err == QUERY_CANCELED || is_dbx_query_timeout_error(&err.to_lowercase()))
{
con.mark_draining();
state.spawn_duckdb_pool_cleanup(pool_key.to_string(), con);
}
result
}
DuckDbTaskWait::Draining { error, task } => {
con.mark_draining();
state.spawn_duckdb_draining_cleanup(pool_key.to_string(), con, task);
Err(error)
}
}
}
#[cfg(feature = "duckdb-bundled")]
PoolKind::DuckDbWorker(client) => { PoolKind::DuckDbWorker(client) => {
let client = client.clone(); let client = client.clone();
if let Some(ref execution_id) = options.execution_id { if let Some(ref execution_id) = options.execution_id {
@@ -1445,11 +1001,7 @@ pub async fn do_execute(
drop(connections); drop(connections);
client.execute(database, sql, max_rows, cancel_token, query_timeout).await client.execute(database, sql, max_rows, cancel_token, query_timeout).await
} }
#[cfg(not(feature = "duckdb-bundled"))] #[cfg(not(feature = "duckdb-sidecar"))]
PoolKind::DuckDb(_) => {
return Err("DuckDB support is not compiled in this build".to_string());
}
#[cfg(not(feature = "duckdb-bundled"))]
PoolKind::DuckDbWorker(_) => { PoolKind::DuckDbWorker(_) => {
return Err("DuckDB worker support is not compiled in this build".to_string()); return Err("DuckDB worker support is not compiled in this build".to_string());
} }
@@ -1719,32 +1271,6 @@ pub async fn do_execute(
} }
result result
} }
#[cfg(feature = "duckdb-bundled")]
PoolKind::ExternalTabular(ext_pool) => {
if !starts_with_duckdb_result_sql_keyword(sql) {
return Err("External data sources are read-only. Only SELECT queries are supported.".to_string());
}
let con = ext_pool.cache.clone();
let interrupt_handle = con.lock().map_err(|e| e.to_string())?.interrupt_handle();
if let Some(ref execution_id) = options.execution_id {
let cancel_interrupt_handle = interrupt_handle.clone();
state.running_queries.register_interrupt(execution_id, move || {
cancel_interrupt_handle.interrupt();
});
}
let sql = sql.to_string();
let max_rows = options.max_rows;
drop(connections);
let task = tokio::task::spawn_blocking(move || {
let con = con.lock().map_err(|e| e.to_string())?;
duckdb_execute_with_max_rows(&con, &sql, max_rows)
});
wait_for_duckdb_task_with_interrupt(cancel_token, query_timeout, interrupt_handle, task).await
}
#[cfg(not(feature = "duckdb-bundled"))]
PoolKind::ExternalTabular(_) => {
Err("External data sources require DuckDB support. Rebuild with default features.".to_string())
}
PoolKind::ExternalDriver { config, session, .. } => { PoolKind::ExternalDriver { config, session, .. } => {
let config = config.clone(); let config = config.clone();
let session = session.clone(); let session = session.clone();
@@ -2769,7 +2295,6 @@ fn schema_diff_failure_outcome(
} }
fn pool_kind_has_transactional_path(pool: &PoolKind) -> bool { fn pool_kind_has_transactional_path(pool: &PoolKind) -> bool {
// DuckDb variants always exist (unit stubs when duckdb-bundled is off).
match pool { match pool {
PoolKind::Postgres(_) PoolKind::Postgres(_)
| PoolKind::Mysql(_, _) | PoolKind::Mysql(_, _)
@@ -2783,14 +2308,12 @@ fn pool_kind_has_transactional_path(pool: &PoolKind) -> bool {
PoolKind::MessageQueue PoolKind::MessageQueue
| PoolKind::Nacos | PoolKind::Nacos
| PoolKind::HBase(_) | PoolKind::HBase(_)
| PoolKind::DuckDb(_)
| PoolKind::DuckDbWorker(_) | PoolKind::DuckDbWorker(_)
| PoolKind::Redis(_) | PoolKind::Redis(_)
| PoolKind::MongoDb(_) | PoolKind::MongoDb(_)
| PoolKind::Elasticsearch(_) | PoolKind::Elasticsearch(_)
| PoolKind::VectorDb(_) | PoolKind::VectorDb(_)
| PoolKind::InfluxDb(_) | PoolKind::InfluxDb(_)
| PoolKind::ExternalTabular(_)
| PoolKind::ExternalDriver { .. } => false, | PoolKind::ExternalDriver { .. } => false,
} }
} }
@@ -2980,25 +2503,12 @@ pub async fn execute_statements_in_transaction_on_pool(
| PoolKind::SqlServer(_) | PoolKind::SqlServer(_)
| PoolKind::Agent(_) => TxPath::Explicit, | PoolKind::Agent(_) => TxPath::Explicit,
PoolKind::MessageQueue | PoolKind::Nacos | PoolKind::HBase(_) => TxPath::None, PoolKind::MessageQueue | PoolKind::Nacos | PoolKind::HBase(_) => TxPath::None,
#[cfg(feature = "duckdb-bundled")] PoolKind::DuckDbWorker(_)
PoolKind::DuckDb(_)
| PoolKind::DuckDbWorker(_)
| PoolKind::Redis(_) | PoolKind::Redis(_)
| PoolKind::MongoDb(_) | PoolKind::MongoDb(_)
| PoolKind::Elasticsearch(_) | PoolKind::Elasticsearch(_)
| PoolKind::VectorDb(_) | PoolKind::VectorDb(_)
| PoolKind::InfluxDb(_) | PoolKind::InfluxDb(_)
| PoolKind::ExternalTabular(_)
| PoolKind::ExternalDriver { .. } => TxPath::None,
#[cfg(not(feature = "duckdb-bundled"))]
PoolKind::DuckDb(_)
| PoolKind::DuckDbWorker(_)
| PoolKind::Redis(_)
| PoolKind::MongoDb(_)
| PoolKind::Elasticsearch(_)
| PoolKind::VectorDb(_)
| PoolKind::InfluxDb(_)
| PoolKind::ExternalTabular(_)
| PoolKind::ExternalDriver { .. } => TxPath::None, | PoolKind::ExternalDriver { .. } => TxPath::None,
}) })
}; };
@@ -4744,192 +4254,6 @@ mod tests {
assert_eq!(result.unwrap_err(), QUERY_CANCELED); assert_eq!(result.unwrap_err(), QUERY_CANCELED);
} }
#[cfg(feature = "duckdb-bundled")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn duckdb_timeout_interrupts_running_task_without_waiting_for_it_to_finish() {
let con = std::sync::Arc::new(crate::db::duckdb_driver::DuckDbConnection::new(
duckdb::Connection::open_in_memory().unwrap(),
));
let interrupt_handle = con.interrupt_handle();
let running_con = con.clone();
let task = tokio::task::spawn_blocking(move || {
let con = running_con.lock().map_err(|e| e.to_string())?;
duckdb_execute_with_max_rows(&con, "SELECT sum(sin(i::DOUBLE)) FROM range(10000000000) tbl(i)", None)
});
let started = std::time::Instant::now();
let result =
wait_for_duckdb_task_with_interrupt(None, Some(Duration::from_millis(10)), interrupt_handle, task).await;
assert_eq!(result.unwrap_err(), timeout_error());
assert!(started.elapsed() < Duration::from_secs(5));
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn duckdb_cancel_keeps_pool_draining_until_references_drop() {
let dir = std::env::temp_dir().join(format!("dbx-query-duckdb-cancel-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let pool_key = "duckdb-1";
let con = std::sync::Arc::new(crate::db::duckdb_driver::DuckDbConnection::new(
duckdb::Connection::open_in_memory().unwrap(),
));
let extra_reference = con.clone();
state.connections.write().await.insert(pool_key.to_string(), PoolKind::DuckDb(con));
state.configs.write().await.insert(pool_key.to_string(), test_connection_config(DatabaseType::DuckDb));
let token = CancellationToken::new();
let cancel_token = token.clone();
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(10)).await;
cancel_token.cancel();
});
let result = do_execute(
&state,
pool_key,
db::mysql::MySqlQueryDialect::default(),
Some("main"),
"SELECT sum(sin(i::DOUBLE)) FROM range(10000000000) tbl(i)",
None,
Some(token),
QueryExecutionOptions::default(),
)
.await;
assert_eq!(result.unwrap_err(), QUERY_CANCELED);
let still_present = {
let conns = state.connections.read().await;
matches!(conns.get(pool_key), Some(PoolKind::DuckDb(current)) if current.is_draining())
};
assert!(still_present);
drop(extra_reference);
// The draining-cleanup task (spawn_duckdb_draining_cleanup) cannot drop
// the pool until the cancelled query's blocking DuckDB task has fully
// unwound — the connection must stay alive while the query still holds
// it. That unwind is near-instant on an idle runner but can exceed the
// original 5s window under heavy CI load, flaking this test, so allow
// ample headroom.
timeout(Duration::from_secs(30), async {
loop {
if !state.connections.read().await.contains_key(pool_key) {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("draining DuckDB pool should be removed after references drop");
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn duckdb_draining_pool_rejects_follow_up_query() {
let dir = std::env::temp_dir().join(format!("dbx-query-duckdb-draining-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let pool_key = "duckdb-1";
let con = std::sync::Arc::new(crate::db::duckdb_driver::DuckDbConnection::new(
duckdb::Connection::open_in_memory().unwrap(),
));
con.mark_draining();
state.connections.write().await.insert(pool_key.to_string(), PoolKind::DuckDb(con));
state.configs.write().await.insert(pool_key.to_string(), test_connection_config(DatabaseType::DuckDb));
let result = do_execute(
&state,
pool_key,
db::mysql::MySqlQueryDialect::default(),
Some("main"),
"SELECT 1",
None,
None,
QueryExecutionOptions::default(),
)
.await;
assert_eq!(result.unwrap_err(), duckdb_draining_error());
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn duckdb_draining_cleanup_removes_pool_after_task_finishes() {
let dir = std::env::temp_dir().join(format!("dbx-query-duckdb-cleanup-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let pool_key = "duckdb-1";
let con = std::sync::Arc::new(crate::db::duckdb_driver::DuckDbConnection::new(
duckdb::Connection::open_in_memory().unwrap(),
));
con.mark_draining();
state.connections.write().await.insert(pool_key.to_string(), PoolKind::DuckDb(con.clone()));
let task_con = con.clone();
let task = tokio::task::spawn_blocking(move || {
let _locked = task_con.lock().map_err(|e| e.to_string())?;
std::thread::sleep(Duration::from_millis(100));
Ok(empty_query_result(0))
});
state.spawn_duckdb_draining_cleanup(pool_key.to_string(), con.clone(), task);
assert!(state.connections.read().await.contains_key(pool_key));
drop(con);
timeout(Duration::from_secs(5), async {
loop {
if !state.connections.read().await.contains_key(pool_key) {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("draining cleanup should remove the DuckDB pool");
}
#[cfg(feature = "duckdb-bundled")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn duckdb_cleanup_keeps_draining_pool_while_extra_reference_exists() {
let dir = std::env::temp_dir().join(format!("dbx-query-duckdb-cleanup-ref-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let pool_key = "duckdb-1";
let con = std::sync::Arc::new(crate::db::duckdb_driver::DuckDbConnection::new(
duckdb::Connection::open_in_memory().unwrap(),
));
con.mark_draining();
state.connections.write().await.insert(pool_key.to_string(), PoolKind::DuckDb(con.clone()));
let extra_reference = con.clone();
let task = tokio::task::spawn_blocking(|| Ok(empty_query_result(0)));
state.spawn_duckdb_draining_cleanup(pool_key.to_string(), con.clone(), task);
tokio::time::sleep(Duration::from_millis(100)).await;
let still_present = {
let conns = state.connections.read().await;
matches!(conns.get(pool_key), Some(PoolKind::DuckDb(current)) if current.is_draining())
};
assert!(still_present);
drop(extra_reference);
drop(con);
timeout(Duration::from_secs(5), async {
loop {
if !state.connections.read().await.contains_key(pool_key) {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("draining cleanup should remove the DuckDB pool after extra refs drop");
}
#[test] #[test]
fn db_operation_budget_from_config() { fn db_operation_budget_from_config() {
let budget = DbOperationBudget::from_config(10, Some(30)); let budget = DbOperationBudget::from_config(10, Some(30));
@@ -5162,150 +4486,6 @@ mod tests {
assert_eq!(pool_error_action(Some(DatabaseType::Dameng), dameng_err), PoolErrorAction::ReconnectAndRetry); assert_eq!(pool_error_action(Some(DatabaseType::Dameng), dameng_err), PoolErrorAction::ReconnectAndRetry);
} }
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_preserves_double_precision() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(
&con,
"SELECT 12.34567::DOUBLE AS sample, 0.5::DOUBLE AS half, 99.99::DOUBLE AS price, 1.0::DOUBLE AS one",
)
.expect("execute double query");
assert_eq!(result.columns, vec!["sample", "half", "price", "one"]);
let row = &result.rows[0];
assert_eq!(row[0], serde_json::json!(12.34567));
assert_eq!(row[1], serde_json::json!(0.5));
assert_eq!(row[2], serde_json::json!(99.99));
assert_eq!(row[3], serde_json::json!(1.0));
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_create_insert_select_double() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
con.execute_batch("CREATE TABLE tmp1 (tmp_double DOUBLE)").expect("create table");
con.execute_batch("INSERT INTO tmp1 VALUES (45.678), (12.345), (99.999)").expect("insert");
let result = duckdb_execute(&con, "SELECT tmp_double FROM tmp1 ORDER BY tmp_double").expect("select doubles");
assert_eq!(result.rows.len(), 3);
assert_eq!(result.rows[0][0], serde_json::json!(12.345));
assert_eq!(result.rows[1][0], serde_json::json!(45.678));
assert_eq!(result.rows[2][0], serde_json::json!(99.999));
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_returns_rows_for_from_first_query() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
con.execute_batch("CREATE TABLE users (id INTEGER, name VARCHAR)").expect("create table");
con.execute_batch("INSERT INTO users VALUES (2, 'Grace'), (1, 'Ada')").expect("insert");
let result = duckdb_execute(&con, "FROM users ORDER BY id").expect("execute from-first query");
assert_eq!(result.columns, vec!["id", "name"]);
assert_eq!(result.rows.len(), 2);
assert_eq!(result.rows[0], vec![serde_json::json!(1), serde_json::json!("Ada")]);
assert_eq!(result.rows[1], vec![serde_json::json!(2), serde_json::json!("Grace")]);
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_returns_rows_for_summarize_query() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
con.execute_batch("CREATE TABLE metrics (value INTEGER)").expect("create table");
con.execute_batch("INSERT INTO metrics VALUES (1), (2), (NULL)").expect("insert");
let result = duckdb_execute(&con, "SUMMARIZE metrics").expect("execute summarize query");
assert!(!result.columns.is_empty());
assert!(!result.rows.is_empty());
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_handles_various_types() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(
&con,
"SELECT 42 AS int_val, true AS bool_val, 'hello' AS text_val, 3.14::FLOAT AS float_val, 123456789012345::BIGINT AS big_val",
)
.expect("execute mixed types query");
let row = &result.rows[0];
assert_eq!(row[0], serde_json::json!(42));
assert_eq!(row[1], serde_json::json!(true));
assert_eq!(row[2], serde_json::Value::String("hello".to_string()));
assert!(row[3].is_number());
assert_eq!(row[4], serde_json::json!(123456789012345_i64));
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_returns_list_values_as_json_arrays() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(&con, "SELECT ['a','b','c','d'];").expect("execute list query");
assert_eq!(result.rows, vec![vec![serde_json::json!(["a", "b", "c", "d"])]]);
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_preserves_nulls_inside_list_values() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(&con, "SELECT [1, NULL, 3] AS items;").expect("execute nullable list query");
assert_eq!(result.columns, vec!["items"]);
assert_eq!(result.rows, vec![vec![serde_json::json!([1, null, 3])]]);
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_returns_nested_complex_values_as_json() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(
&con,
"SELECT {'name': 'Ada', 'scores': [10, 20]} AS profile, MAP(['x', 'y'], [1, 2]) AS lookup, [1, 2, 3]::INTEGER[3] AS fixed_items",
)
.expect("execute complex values query");
assert_eq!(result.columns, vec!["profile", "lookup", "fixed_items"]);
assert_eq!(
result.rows,
vec![vec![
serde_json::json!({ "name": "Ada", "scores": [10, 20] }),
serde_json::json!([
{ "key": "x", "value": 1 },
{ "key": "y", "value": 2 },
]),
serde_json::json!([1, 2, 3]),
]]
);
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_formats_temporal_values_by_column_type() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(
&con,
"SELECT DATE '2026-05-14' AS d, TIME '16:58:15' AS t, TIMESTAMP '2026-05-14 16:58:15.0' AS ts, NULL::TIMESTAMP AS nts",
)
.expect("execute temporal query");
assert_eq!(result.columns, vec!["d", "t", "ts", "nts"]);
assert_eq!(
result.rows,
vec![vec![
serde_json::Value::String("2026-05-14".to_string()),
serde_json::Value::String("16:58:15".to_string()),
serde_json::Value::String("2026-05-14 16:58:15".to_string()),
serde_json::Value::Null,
]]
);
}
#[test] #[test]
fn external_driver_query_params_include_database_and_schema_context() { fn external_driver_query_params_include_database_and_schema_context() {
let config = ConnectionConfig { let config = ConnectionConfig {
File diff suppressed because it is too large. Load diff
@@ -1,203 +0,0 @@
use crate::connection::AppState;
use crate::db;
pub fn duckdb_query_tables(con: &duckdb::Connection) -> Result<Vec<db::TableInfo>, String> {
duckdb_query_tables_in_database(con, "main", "main")
}
pub fn duckdb_query_tables_in_database(
con: &duckdb::Connection,
database: &str,
schema: &str,
) -> Result<Vec<db::TableInfo>, String> {
duckdb_query_tables_in_database_with_attached(con, database, schema, &[])
}
pub fn duckdb_query_tables_in_database_with_attached(
con: &duckdb::Connection,
database: &str,
schema: &str,
attached_names: &[String],
) -> Result<Vec<db::TableInfo>, String> {
let database = duckdb_catalog_name(con, database, attached_names)?;
let mut stmt = con
.prepare(
"SELECT table_name, table_type FROM information_schema.tables WHERE table_catalog = ? AND table_schema = ? ORDER BY table_name",
)
.map_err(|e| e.to_string())?;
let rows = stmt
.query_map((database.as_str(), schema), |row| {
Ok(db::TableInfo {
name: row.get::<_, String>(0)?,
table_type: row.get::<_, String>(1)?,
comment: None,
parent_schema: None,
parent_name: None,
})
})
.map_err(|e| e.to_string())?;
Ok(rows.filter_map(|r| r.ok()).collect())
}
pub fn duckdb_attach_database(con: &duckdb::Connection, name: &str, path: &str) -> Result<(), String> {
let name = name.trim();
let path = path.trim();
if name.is_empty() || path.is_empty() {
return Err("DuckDB attached database name and path are required".to_string());
}
let sql = format!("ATTACH {} AS {}", duckdb_quote_string(path), duckdb_quote_ident(name));
con.execute_batch(&sql).map_err(|e| e.to_string())
}
pub fn duckdb_list_databases(con: &duckdb::Connection) -> Result<Vec<db::DatabaseInfo>, String> {
duckdb_list_databases_with_attached(con, &[])
}
pub fn duckdb_list_databases_with_attached(
con: &duckdb::Connection,
attached_names: &[String],
) -> Result<Vec<db::DatabaseInfo>, String> {
let primary = duckdb_primary_catalog(con, attached_names)?;
let mut stmt = con.prepare("SHOW DATABASES").map_err(|e| e.to_string())?;
let rows = stmt
.query_map([], |row| {
let name = row.get::<_, String>(0)?;
Ok(db::DatabaseInfo { name: if name == primary { "main".to_string() } else { name } })
})
.map_err(|e| e.to_string())?;
Ok(rows.filter_map(|row| row.ok()).collect())
}
pub fn duckdb_list_schemas(con: &duckdb::Connection, database: &str) -> Result<Vec<String>, String> {
duckdb_list_schemas_with_attached(con, database, &[])
}
pub fn duckdb_list_schemas_with_attached(
con: &duckdb::Connection,
database: &str,
attached_names: &[String],
) -> Result<Vec<String>, String> {
let database = duckdb_catalog_name(con, database, attached_names)?;
let mut stmt = con
.prepare(
"SELECT schema_name FROM information_schema.schemata WHERE catalog_name = ? AND schema_name NOT IN ('information_schema', 'pg_catalog') ORDER BY schema_name",
)
.map_err(|e| e.to_string())?;
let rows = stmt.query_map([database.as_str()], |row| row.get::<_, String>(0)).map_err(|e| e.to_string())?;
Ok(rows.filter_map(|r| r.ok()).collect())
}
fn duckdb_catalog_name(con: &duckdb::Connection, database: &str, attached_names: &[String]) -> Result<String, String> {
if database.trim().is_empty() || database == "main" {
return duckdb_primary_catalog(con, attached_names);
}
Ok(database.to_string())
}
pub fn duckdb_primary_catalog(con: &duckdb::Connection, attached_names: &[String]) -> Result<String, String> {
if attached_names.is_empty() {
return duckdb_current_database(con);
}
let attached: std::collections::HashSet<String> = attached_names.iter().map(|name| name.to_lowercase()).collect();
let mut stmt = con.prepare("SHOW DATABASES").map_err(|e| e.to_string())?;
let rows = stmt.query_map([], |row| row.get::<_, String>(0)).map_err(|e| e.to_string())?;
for row in rows {
let name = row.map_err(|e| e.to_string())?;
if !attached.contains(&name.to_lowercase()) {
return Ok(name);
}
}
duckdb_current_database(con)
}
fn duckdb_current_database(con: &duckdb::Connection) -> Result<String, String> {
con.query_row("SELECT current_database()", [], |row| row.get::<_, String>(0)).map_err(|e| e.to_string())
}
fn duckdb_quote_ident(value: &str) -> String {
format!("\"{}\"", value.replace('"', "\"\""))
}
fn duckdb_quote_string(value: &str) -> String {
format!("'{}'", value.replace('\'', "''"))
}
pub fn duckdb_query_columns(con: &duckdb::Connection, table: &str) -> Result<Vec<db::ColumnInfo>, String> {
duckdb_query_columns_in_database(con, "main", "main", table)
}
pub fn duckdb_query_columns_in_database(
con: &duckdb::Connection,
database: &str,
schema: &str,
table: &str,
) -> Result<Vec<db::ColumnInfo>, String> {
duckdb_query_columns_in_database_with_attached(con, database, schema, table, &[])
}
pub fn duckdb_query_columns_in_database_with_attached(
con: &duckdb::Connection,
database: &str,
schema: &str,
table: &str,
attached_names: &[String],
) -> Result<Vec<db::ColumnInfo>, String> {
let database = duckdb_catalog_name(con, database, attached_names)?;
let mut pk_stmt = con
.prepare(
"SELECT kcu.column_name
FROM information_schema.table_constraints tc
JOIN information_schema.key_column_usage kcu
ON tc.constraint_name = kcu.constraint_name
AND tc.table_schema = kcu.table_schema
AND tc.table_name = kcu.table_name
WHERE tc.constraint_type = 'PRIMARY KEY'
AND tc.table_catalog = ?
AND tc.table_schema = ?
AND tc.table_name = ?
ORDER BY kcu.ordinal_position",
)
.map_err(|e| e.to_string())?;
let pk_rows = pk_stmt
.query_map((database.as_str(), schema, table), |row| row.get::<_, String>(0))
.map_err(|e| e.to_string())?;
let primary_keys: std::collections::HashSet<String> = pk_rows.filter_map(|r| r.ok()).collect();
let mut stmt = con
.prepare(
"SELECT column_name, data_type, is_nullable, column_default
FROM information_schema.columns
WHERE table_catalog = ? AND table_schema = ? AND table_name = ?
ORDER BY ordinal_position",
)
.map_err(|e| e.to_string())?;
let rows = stmt
.query_map((database.as_str(), schema, table), |row| {
let name = row.get::<_, String>(0)?;
Ok(db::ColumnInfo {
is_primary_key: primary_keys.contains(&name),
name,
data_type: row.get::<_, String>(1)?,
is_nullable: row.get::<_, String>(2).unwrap_or_default() == "YES",
column_default: row.get::<_, Option<String>>(3)?,
extra: None,
comment: None,
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
..Default::default()
})
})
.map_err(|e| e.to_string())?;
Ok(rows.filter_map(|r| r.ok()).collect())
}
pub(super) async fn duckdb_attached_database_names(state: &AppState, connection_id: &str) -> Vec<String> {
state
.configs
.read()
.await
.get(connection_id)
.map(crate::db::duckdb_sql::config_attached_names)
.unwrap_or_default()
}
@@ -6,7 +6,7 @@ pub(super) enum StructureDialect {
Doris, Doris,
Postgres, Postgres,
Sqlite, Sqlite,
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
DuckDb, DuckDb,
SqlServer, SqlServer,
Oracle, Oracle,
@@ -171,7 +171,7 @@ pub(super) fn capabilities_for(database_type: Option<DatabaseType>) -> TableStru
index_filter: true, index_filter: true,
..base ..base
}, },
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
Some(DatabaseType::DuckDb) => TableStructureCapabilities { Some(DatabaseType::DuckDb) => TableStructureCapabilities {
dialect: StructureDialect::DuckDb, dialect: StructureDialect::DuckDb,
add_column: true, add_column: true,
@@ -305,7 +305,7 @@ pub(super) fn dialect_label(dialect: StructureDialect) -> String {
StructureDialect::Doris => "doris", StructureDialect::Doris => "doris",
StructureDialect::Postgres => "postgres", StructureDialect::Postgres => "postgres",
StructureDialect::Sqlite => "sqlite", StructureDialect::Sqlite => "sqlite",
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
StructureDialect::DuckDb => "duckdb", StructureDialect::DuckDb => "duckdb",
StructureDialect::SqlServer => "sqlserver", StructureDialect::SqlServer => "sqlserver",
StructureDialect::Oracle => "oracle", StructureDialect::Oracle => "oracle",
@@ -326,7 +326,7 @@ pub(super) fn database_type_for_dialect(dialect: StructureDialect) -> Option<Dat
StructureDialect::Doris => Some(DatabaseType::Doris), StructureDialect::Doris => Some(DatabaseType::Doris),
StructureDialect::Postgres => Some(DatabaseType::Postgres), StructureDialect::Postgres => Some(DatabaseType::Postgres),
StructureDialect::Sqlite => Some(DatabaseType::Sqlite), StructureDialect::Sqlite => Some(DatabaseType::Sqlite),
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
StructureDialect::DuckDb => Some(DatabaseType::DuckDb), StructureDialect::DuckDb => Some(DatabaseType::DuckDb),
StructureDialect::SqlServer => Some(DatabaseType::SqlServer), StructureDialect::SqlServer => Some(DatabaseType::SqlServer),
StructureDialect::Oracle => Some(DatabaseType::Oracle), StructureDialect::Oracle => Some(DatabaseType::Oracle),
@@ -2465,7 +2465,7 @@ fn sqlserver_existing_column_identity_change_warns_without_unchanged_foreign_key
); );
} }
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
#[test] #[test]
fn builds_duckdb_create_table_statements() { fn builds_duckdb_create_table_statements() {
let mut name = column("name"); let mut name = column("name");
+10 -193
View File
@@ -10,8 +10,6 @@ use crate::models::connection::DatabaseType;
use crate::object_source_sql::{build_executable_object_source_statements, EditableObjectSourceSqlInput}; use crate::object_source_sql::{build_executable_object_source_statements, EditableObjectSourceSqlInput};
use crate::query::{agent_execute_query_params, pool_error_action, PoolErrorAction, QueryExecutionOptions}; use crate::query::{agent_execute_query_params, pool_error_action, PoolErrorAction, QueryExecutionOptions};
use crate::sql::split_sql_statements; use crate::sql::split_sql_statements;
#[cfg(feature = "duckdb-bundled")]
use crate::sql::starts_with_executable_sql_keyword;
use crate::sql_dialect::{qualified_transfer_table, quote_transfer_identifier}; use crate::sql_dialect::{qualified_transfer_table, quote_transfer_identifier};
static CANCELLED: std::sync::LazyLock<RwLock<HashSet<String>>> = static CANCELLED: std::sync::LazyLock<RwLock<HashSet<String>>> =
@@ -3048,89 +3046,12 @@ async fn execute_on_pool_once(
); );
client.execute_query(params).await client.execute_query(params).await
} }
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
PoolKind::DuckDb(con) => { PoolKind::DuckDbWorker(client) => {
let con = con.clone(); let client = client.clone();
let sql = sql.to_string(); let sql = sql.to_string();
drop(connections); drop(connections);
tokio::task::spawn_blocking(move || { client.execute(None, sql, max_rows, None, None).await
let con = con.lock().map_err(|e| e.to_string())?;
if max_rows.is_some()
&& starts_with_executable_sql_keyword(&sql, &["SELECT", "SHOW", "DESCRIBE", "WITH", "PRAGMA"])
{
return crate::query::duckdb_execute_with_max_rows(&con, &sql, max_rows);
}
let start = std::time::Instant::now();
if starts_with_executable_sql_keyword(&sql, &["SELECT", "SHOW", "DESCRIBE", "WITH", "PRAGMA"]) {
let mut stmt = con.prepare(&sql).map_err(|e| e.to_string())?;
let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
let stmt_ref = rows.as_ref().ok_or("DuckDB statement unavailable")?;
let col_count = stmt_ref.column_count();
let columns: Vec<String> = (0..col_count)
.map(|i| stmt_ref.column_name(i).map(|s| s.to_string()).unwrap_or_else(|_| "?".to_string()))
.collect();
let mut result_rows = Vec::new();
while let Some(row) = rows.next().map_err(|e| e.to_string())? {
let vals: Vec<serde_json::Value> = (0..col_count)
.map(|i| {
row.get::<_, String>(i)
.map(serde_json::Value::String)
.or_else(|_| row.get::<_, i64>(i).map(|v| serde_json::Value::Number(v.into())))
.or_else(|_| {
row.get::<_, f64>(i).map(|v| {
serde_json::Number::from_f64(v)
.map(serde_json::Value::Number)
.unwrap_or(serde_json::Value::Null)
})
})
.or_else(|_| row.get::<_, bool>(i).map(serde_json::Value::Bool))
.unwrap_or(serde_json::Value::Null)
})
.collect();
result_rows.push(vals);
}
Ok(db::QueryResult {
columns,
column_types: Vec::new(),
column_sortables: vec![],
rows: result_rows,
affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(),
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
})
} else {
let affected = con.execute(&sql, []).map_err(|e| e.to_string())?;
Ok(db::QueryResult {
columns: vec![],
column_types: Vec::new(),
column_sortables: vec![],
rows: vec![],
affected_rows: affected as u64,
execution_time_ms: start.elapsed().as_millis(),
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
})
}
})
.await
.map_err(|e| e.to_string())?
}
#[cfg(feature = "duckdb-bundled")]
PoolKind::ExternalTabular(ext_pool) => {
let con = ext_pool.cache.clone();
let sql = sql.to_string();
drop(connections);
tokio::task::spawn_blocking(move || {
let con = con.lock().map_err(|e| e.to_string())?;
crate::query::duckdb_execute_with_max_rows(&con, &sql, max_rows)
})
.await
.map_err(|e| e.to_string())?
} }
_ => Err("Unsupported database type for transfer".to_string()), _ => Err("Unsupported database type for transfer".to_string()),
} }
@@ -3161,32 +3082,14 @@ pub async fn get_columns_for_transfer(
) -> Result<Vec<db::ColumnInfo>, String> { ) -> Result<Vec<db::ColumnInfo>, String> {
let connections = state.connections.read().await; let connections = state.connections.read().await;
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
if let Some(PoolKind::DuckDb(con)) = connections.get(pool_key) { if let Some(PoolKind::DuckDbWorker(client)) = connections.get(pool_key) {
let con = con.clone(); let client = client.clone();
drop(connections); let database = database.to_string();
let table = table.to_string();
let schema = schema.to_string(); let schema = schema.to_string();
return tokio::task::spawn_blocking(move || {
let con = con.lock().map_err(|e| e.to_string())?;
crate::schema::duckdb_query_columns_in_database(&con, "main", &schema, &table)
})
.await
.map_err(|e| e.to_string())?;
}
#[cfg(feature = "duckdb-bundled")]
if let Some(PoolKind::ExternalTabular(ext_pool)) = connections.get(pool_key) {
let con = ext_pool.cache.clone();
drop(connections);
let table = table.to_string(); let table = table.to_string();
let schema = schema.to_string(); drop(connections);
return tokio::task::spawn_blocking(move || { return client.list_columns(database, schema, table).await;
let con = con.lock().map_err(|e| e.to_string())?;
crate::schema::duckdb_query_columns_in_database(&con, "main", &schema, &table)
})
.await
.map_err(|e| e.to_string())?;
} }
if let Some(PoolKind::ClickHouse(client)) = connections.get(pool_key) { if let Some(PoolKind::ClickHouse(client)) = connections.get(pool_key) {
@@ -5029,73 +4932,7 @@ where
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
#[cfg(feature = "duckdb-bundled")]
use crate::connection::{AppState, PoolKind};
#[cfg(feature = "duckdb-bundled")]
use crate::models::connection::default_redis_key_separator;
#[cfg(feature = "duckdb-bundled")]
use crate::storage::Storage;
use serde_json::json; use serde_json::json;
#[cfg(feature = "duckdb-bundled")]
use std::sync::Arc;
#[cfg(feature = "duckdb-bundled")]
fn duckdb_test_config(id: &str) -> crate::models::connection::ConnectionConfig {
crate::models::connection::ConnectionConfig {
id: id.to_string(),
name: id.to_string(),
note: String::new(),
db_type: DatabaseType::DuckDb,
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: ":memory:".to_string(),
port: 0,
username: String::new(),
password: String::new(),
database: None,
visible_databases: None,
visible_schemas: None,
show_system_schemas: false,
attached_databases: Vec::new(),
init_script: None,
color: None,
transport_layers: Vec::new(),
connect_timeout_secs: 5,
query_timeout_secs: 30,
idle_timeout_secs: 60,
keepalive_interval_secs: 0,
ssl: false,
ca_cert_path: String::new(),
client_cert_path: String::new(),
client_key_path: String::new(),
sysdba: false,
oracle_connection_type: None,
connection_string: None,
redis_connection_mode: None,
redis_sentinel_master: String::new(),
redis_sentinel_nodes: String::new(),
redis_sentinel_username: String::new(),
redis_sentinel_password: String::new(),
redis_sentinel_tls: false,
redis_cluster_nodes: String::new(),
redis_key_separator: default_redis_key_separator(),
redis_scan_page_size: None,
redis_database_aliases: Default::default(),
etcd_endpoints: String::new(),
gbase_server: String::new(),
informix_server: String::new(),
external_config: None,
jdbc_driver_class: None,
jdbc_driver_paths: Vec::new(),
one_time: false,
read_only: false,
is_production: false,
production_databases: vec![],
database_info: None,
}
}
fn test_column(name: &str, data_type: &str) -> db::ColumnInfo { fn test_column(name: &str, data_type: &str) -> db::ColumnInfo {
db::ColumnInfo { db::ColumnInfo {
@@ -6863,26 +6700,6 @@ SELECT 1 FROM dual"#
assert!(statements[0].contains("ON DUPLICATE KEY UPDATE")); assert!(statements[0].contains("ON DUPLICATE KEY UPDATE"));
} }
#[cfg(feature = "duckdb-bundled")]
#[tokio::test]
async fn duckdb_transfer_columns_use_requested_schema() {
let dir = std::env::temp_dir().join(format!("dbx-transfer-test-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let con = duckdb::Connection::open_in_memory().unwrap();
con.execute_batch("CREATE SCHEMA analytics; CREATE TABLE analytics.items(id INTEGER);").unwrap();
let state = AppState::new(storage);
let con = Arc::new(crate::db::duckdb_driver::DuckDbConnection::new(con));
state.connections.write().await.insert("duckdb-1".to_string(), PoolKind::DuckDb(con));
state.configs.write().await.insert("duckdb-1".to_string(), duckdb_test_config("duckdb-1"));
let columns =
get_columns_for_transfer(&state, "duckdb-1", "duckdb-1", "main", "analytics", "items").await.unwrap();
assert_eq!(columns.iter().map(|c| c.name.as_str()).collect::<Vec<_>>(), vec!["id"]);
}
#[test] #[test]
fn database_from_pool_key_handles_session_scoped_keys() { fn database_from_pool_key_handles_session_scoped_keys() {
assert_eq!(database_from_pool_key("conn:analytics"), Some("analytics")); assert_eq!(database_from_pool_key("conn:analytics"), Some("analytics"));
+196 -4
View File
@@ -4,8 +4,9 @@ use dbx_core::agent_manager::{
}; };
use dbx_core::agent_service::{ use dbx_core::agent_service::{
build_agent_list, clear_agent_download_cache, github_url_to_r2_path, import_agent_driver, import_agent_jar, build_agent_list, clear_agent_download_cache, github_url_to_r2_path, import_agent_driver, import_agent_jar,
import_agents_from_zip, inspect_offline_zip, is_app_version_compatible, jre_needs_install, import_agents_from_package, import_agents_from_zip, inspect_offline_package, inspect_offline_zip,
local_agent_jar_candidates, replace_download, uninstall_agent_driver, AgentProgressEvent, is_app_version_compatible, jre_needs_install, local_agent_jar_candidates, replace_download, uninstall_agent_driver,
AgentProgressEvent,
}; };
fn test_manager(name: &str) -> AgentManager { fn test_manager(name: &str) -> AgentManager {
@@ -22,7 +23,11 @@ fn registry_with_driver(db_type: &str, version: &str, jre: &str) -> AgentRegistr
label: db_type.to_string(), label: db_type.to_string(),
min_app_version: "0.1.0".to_string(), min_app_version: "0.1.0".to_string(),
jre: jre.to_string(), jre: jre.to_string(),
jar: Some(ArtifactInfo { url: format!("https://example.com/dbx-agent-{db_type}.jar"), size: 42 }), jar: Some(ArtifactInfo {
url: format!("https://example.com/dbx-agent-{db_type}.jar"),
size: 42,
format: None,
}),
native: std::collections::HashMap::new(), native: std::collections::HashMap::new(),
}, },
); );
@@ -50,10 +55,11 @@ fn registry_with_native_driver(db_type: &str, version: &str, jre: &str) -> Agent
jar: Some(ArtifactInfo { jar: Some(ArtifactInfo {
url: format!("https://example.com/dbx-agent-{db_type}-legacy-placeholder.jar"), url: format!("https://example.com/dbx-agent-{db_type}-legacy-placeholder.jar"),
size: 0, size: 0,
format: None,
}), }),
native: [( native: [(
AgentManager::current_platform().to_string(), AgentManager::current_platform().to_string(),
ArtifactInfo { url: format!("https://example.com/dbx-agent-{db_type}"), size: 42 }, ArtifactInfo { url: format!("https://example.com/dbx-agent-{db_type}"), size: 42, format: None },
)] )]
.into_iter() .into_iter()
.collect(), .collect(),
@@ -562,6 +568,21 @@ async fn offline_zip_import_installs_release_named_jre() {
assert!(events.lock().unwrap().iter().any(|event| event.step == "jre-extract")); assert!(events.lock().unwrap().iter().any(|event| event.step == "jre-extract"));
} }
#[tokio::test]
async fn offline_zip_import_installs_tar_zstd_jre() {
let manager = test_manager("offline-zstd-jre");
let zip_path = test_path("offline-zstd-jre-zip").join("agents.zip");
std::fs::create_dir_all(zip_path.parent().unwrap()).unwrap();
write_offline_driver_zip_with_zstd_jre(&zip_path, "h2", "0.2.0", "21.0.12");
let result = import_agents_from_zip(&manager, &zip_path, |_| {}).await.unwrap();
assert_eq!(result.jre_installed, vec![DEFAULT_JRE_KEY]);
assert_eq!(result.drivers_installed, vec!["h2"]);
assert!(manager.is_jre_installed(DEFAULT_JRE_KEY));
assert_eq!(manager.load_state().jre_versions.get(DEFAULT_JRE_KEY).map(String::as_str), Some("21.0.12"));
}
#[tokio::test] #[tokio::test]
async fn offline_zip_import_preserves_existing_jre_when_archive_is_corrupt() { async fn offline_zip_import_preserves_existing_jre_when_archive_is_corrupt() {
let manager = test_manager("offline-corrupt-jre-preserves-existing"); let manager = test_manager("offline-corrupt-jre-preserves-existing");
@@ -688,6 +709,41 @@ async fn offline_zip_import_installs_versioned_native_driver_package() {
assert!(!manager.driver_jar_path("kingbase").exists()); assert!(!manager.driver_jar_path("kingbase").exists());
} }
#[tokio::test]
async fn offline_tar_zstd_import_installs_versioned_native_driver_package() {
let manager = test_manager("offline-tar-zstd-driver-package");
let package_path = test_path("offline-tar-zstd-driver-package").join("duckdb.tar.zst");
std::fs::create_dir_all(package_path.parent().unwrap()).unwrap();
write_offline_tar_zstd_driver_package(&package_path, "duckdb", "0.1.0");
let plan = inspect_offline_package(&package_path).unwrap();
let result = import_agents_from_package(&manager, &package_path, |_| {}).await.unwrap();
assert_eq!(plan.driver_keys, vec!["duckdb"]);
assert!(!plan.includes_jre);
assert_eq!(result.drivers_installed, vec!["duckdb"]);
assert_eq!(manager.load_state().installed_drivers["duckdb"].version, "0.1.0");
assert_eq!(std::fs::read(manager.driver_native_path("duckdb")).unwrap(), current_platform_native_binary());
}
#[tokio::test]
async fn offline_tar_zstd_import_installs_versioned_java_driver_package() {
let manager = test_manager("offline-tar-zstd-java-package");
let package_path = test_path("offline-tar-zstd-java-package").join("dameng.tar.zst");
std::fs::create_dir_all(package_path.parent().unwrap()).unwrap();
write_offline_tar_zstd_java_driver_package(&package_path, "dameng", "0.2.0");
let plan = inspect_offline_package(&package_path).unwrap();
let result = import_agents_from_package(&manager, &package_path, |_| {}).await.unwrap();
assert_eq!(plan.driver_keys, vec!["dameng"]);
assert!(!plan.includes_jre);
assert_eq!(result.drivers_installed, vec!["dameng"]);
assert_eq!(manager.load_state().installed_drivers["dameng"].version, "0.2.0");
assert_eq!(std::fs::read(manager.driver_jar_path("dameng")).unwrap(), test_agent_jar_bytes());
assert!(!manager.driver_native_path("dameng").exists());
}
#[tokio::test] #[tokio::test]
async fn offline_zip_import_rejects_native_driver_for_another_platform() { async fn offline_zip_import_rejects_native_driver_for_another_platform() {
let manager = test_manager("offline-wrong-platform-native-driver-zip"); let manager = test_manager("offline-wrong-platform-native-driver-zip");
@@ -823,6 +879,86 @@ fn write_offline_native_driver_zip(path: &std::path::Path, db_type: &str, versio
write_offline_native_driver_zip_for_platform(path, db_type, version, AgentManager::current_platform()); write_offline_native_driver_zip_for_platform(path, db_type, version, AgentManager::current_platform());
} }
fn write_offline_tar_zstd_driver_package(path: &std::path::Path, db_type: &str, version: &str) {
let native = current_platform_native_binary();
let platform = AgentManager::current_platform();
let extension = if platform.starts_with("windows-") { ".exe" } else { "" };
let filename = format!("dbx-agent-{db_type}-{version}-{platform}{extension}");
let registry = serde_json::json!({
"jres": {},
"drivers": {
db_type: {
"version": version,
"label": db_type,
"min_app_version": "0.6.0",
"jre": DEFAULT_JRE_KEY,
"native": {
platform: {
"url": filename,
"size": native.len()
}
}
}
}
});
let registry_bytes = registry.to_string().into_bytes();
let file = std::fs::File::create(path).unwrap();
let encoder = zstd::stream::write::Encoder::new(file, 3).unwrap();
let mut archive = tar::Builder::new(encoder);
let mut registry_header = tar::Header::new_gnu();
registry_header.set_size(registry_bytes.len() as u64);
registry_header.set_mode(0o644);
registry_header.set_cksum();
archive.append_data(&mut registry_header, "agent-registry.json", registry_bytes.as_slice()).unwrap();
let mut driver_header = tar::Header::new_gnu();
driver_header.set_size(native.len() as u64);
driver_header.set_mode(0o755);
driver_header.set_cksum();
archive.append_data(&mut driver_header, format!("drivers/{filename}"), native.as_slice()).unwrap();
archive.into_inner().unwrap().finish().unwrap();
}
fn write_offline_tar_zstd_java_driver_package(path: &std::path::Path, db_type: &str, version: &str) {
let jar = test_agent_jar_bytes();
let filename = format!("dbx-agent-{db_type}-{version}.jar");
let registry = serde_json::json!({
"jres": {},
"drivers": {
db_type: {
"version": version,
"label": db_type,
"min_app_version": "0.6.0",
"jre": DEFAULT_JRE_KEY,
"jar": {
"url": filename,
"size": jar.len()
}
}
}
});
let registry_bytes = registry.to_string().into_bytes();
let file = std::fs::File::create(path).unwrap();
let encoder = zstd::stream::write::Encoder::new(file, 3).unwrap();
let mut archive = tar::Builder::new(encoder);
let mut registry_header = tar::Header::new_gnu();
registry_header.set_size(registry_bytes.len() as u64);
registry_header.set_mode(0o644);
registry_header.set_cksum();
archive.append_data(&mut registry_header, "agent-registry.json", registry_bytes.as_slice()).unwrap();
let mut driver_header = tar::Header::new_gnu();
driver_header.set_size(jar.len() as u64);
driver_header.set_mode(0o644);
driver_header.set_cksum();
archive.append_data(&mut driver_header, format!("drivers/{filename}"), jar.as_slice()).unwrap();
archive.into_inner().unwrap().finish().unwrap();
}
fn write_offline_native_driver_zip_for_platform(path: &std::path::Path, db_type: &str, version: &str, platform: &str) { fn write_offline_native_driver_zip_for_platform(path: &std::path::Path, db_type: &str, version: &str, platform: &str) {
write_offline_native_driver_zip_with_bytes(path, db_type, version, platform, current_platform_native_binary()); write_offline_native_driver_zip_with_bytes(path, db_type, version, platform, current_platform_native_binary());
} }
@@ -903,6 +1039,46 @@ fn write_offline_driver_zip_with_jre(path: &std::path::Path, db_type: &str, vers
write_offline_driver_zip_with_jre_bytes(path, db_type, version, jre_version, test_jre_archive_bytes()); write_offline_driver_zip_with_jre_bytes(path, db_type, version, jre_version, test_jre_archive_bytes());
} }
fn write_offline_driver_zip_with_zstd_jre(path: &std::path::Path, db_type: &str, version: &str, jre_version: &str) {
let file = std::fs::File::create(path).unwrap();
let mut zip = zip::ZipWriter::new(file);
let options = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored);
let jar = test_agent_jar_bytes();
let jre_archive = test_jre_archive_zstd_bytes();
let platform = AgentManager::current_platform();
let registry = serde_json::json!({
"jres": {
DEFAULT_JRE_KEY: {
"version": jre_version,
"platforms": {
platform: {
"url": format!("https://example.com/dbx-jre-{DEFAULT_JRE_KEY}-{platform}.tar.zst"),
"size": jre_archive.len(),
"format": "tar_zstd"
}
}
}
},
"drivers": {
db_type: {
"version": version,
"label": db_type,
"min_app_version": "0.1.0",
"jre": DEFAULT_JRE_KEY,
"jar": { "url": format!("https://example.com/dbx-agent-{db_type}-{version}.jar"), "size": jar.len() }
}
}
});
zip.start_file("agent-registry.json", options).unwrap();
std::io::Write::write_all(&mut zip, registry.to_string().as_bytes()).unwrap();
zip.start_file(format!("jre/dbx-jre-{DEFAULT_JRE_KEY}-{platform}.tar.zst"), options).unwrap();
std::io::Write::write_all(&mut zip, &jre_archive).unwrap();
zip.start_file(format!("drivers/dbx-agent-{db_type}-{version}.jar"), options).unwrap();
std::io::Write::write_all(&mut zip, &jar).unwrap();
zip.finish().unwrap();
}
fn write_offline_driver_zip_with_jre_bytes( fn write_offline_driver_zip_with_jre_bytes(
path: &std::path::Path, path: &std::path::Path,
db_type: &str, db_type: &str,
@@ -975,6 +1151,22 @@ fn test_jre_archive_bytes() -> Vec<u8> {
bytes bytes
} }
fn test_jre_archive_zstd_bytes() -> Vec<u8> {
let root = test_path("jre-zstd-archive");
let runtime_root = root.join("dbx-jre");
let bin_dir = runtime_root.join("bin");
std::fs::create_dir_all(&bin_dir).unwrap();
std::fs::write(bin_dir.join("java"), b"java").unwrap();
std::fs::write(bin_dir.join("java.exe"), b"java").unwrap();
let encoder = zstd::stream::write::Encoder::new(Vec::new(), 3).unwrap();
let mut builder = tar::Builder::new(encoder);
builder.append_dir_all("dbx-jre", &runtime_root).unwrap();
let bytes = builder.into_inner().unwrap().finish().unwrap();
std::fs::remove_dir_all(root).ok();
bytes
}
fn write_offline_driver_zip_with_jar(path: &std::path::Path, db_type: &str, version: &str, jar: Vec<u8>) { fn write_offline_driver_zip_with_jar(path: &std::path::Path, db_type: &str, version: &str, jar: Vec<u8>) {
let file = std::fs::File::create(path).unwrap(); let file = std::fs::File::create(path).unwrap();
let mut zip = zip::ZipWriter::new(file); let mut zip = zip::ZipWriter::new(file);
@@ -1,9 +0,0 @@
#![cfg(feature = "duckdb-bundled")]
fn main() {
let runtime = tokio::runtime::Runtime::new().expect("Failed to create DuckDB worker test runtime");
if let Err(err) = runtime.block_on(dbx_core::db::duckdb_worker_runtime::run_stdio_worker()) {
eprintln!("DuckDB worker test host failed: {err}");
std::process::exit(1);
}
}
+2 -2
View File
@@ -9,8 +9,8 @@ name = "dbx-web"
path = "src/main.rs" path = "src/main.rs"
[features] [features]
default = ["duckdb-bundled", "mq-admin", "sqlite-sqlcipher", "system-fonts"] default = ["duckdb-sidecar", "mq-admin", "sqlite-sqlcipher", "system-fonts"]
duckdb-bundled = ["dbx-core/duckdb-bundled"] duckdb-sidecar = ["dbx-core/duckdb-sidecar"]
mq-admin = ["dbx-core/mq-admin"] mq-admin = ["dbx-core/mq-admin"]
sqlite-sqlcipher = ["dbx-core/sqlite-sqlcipher"] sqlite-sqlcipher = ["dbx-core/sqlite-sqlcipher"]
system-fonts = ["dbx-core/system-fonts"] system-fonts = ["dbx-core/system-fonts"]
+3 -2
View File
@@ -328,6 +328,7 @@ async fn main() {
.route("/agents/runtime/restart", post(routes::agents::restart_driver_runtime)) .route("/agents/runtime/restart", post(routes::agents::restart_driver_runtime))
.route("/agents/install", post(routes::agents::install_agent)) .route("/agents/install", post(routes::agents::install_agent))
.route("/agents/upgrade-all", post(routes::agents::upgrade_all_agents)) .route("/agents/upgrade-all", post(routes::agents::upgrade_all_agents))
.route("/agents/update-blockers", post(routes::agents::check_agent_update_blockers))
.route("/agents/uninstall", post(routes::agents::uninstall_agent)) .route("/agents/uninstall", post(routes::agents::uninstall_agent))
.route("/agents/import-offline", post(routes::agents::import_agents_from_zip)) .route("/agents/import-offline", post(routes::agents::import_agents_from_zip))
.route("/agents/import-driver", post(routes::agents::import_agent_driver_file)) .route("/agents/import-driver", post(routes::agents::import_agent_driver_file))
@@ -747,8 +748,8 @@ async fn main() {
.route("/cloud-sync/snippet/upload", post(routes::cloud_sync::snippet_sync_upload)) .route("/cloud-sync/snippet/upload", post(routes::cloud_sync::snippet_sync_upload))
.route("/cloud-sync/snippet/download", post(routes::cloud_sync::snippet_sync_download)); .route("/cloud-sync/snippet/download", post(routes::cloud_sync::snippet_sync_download));
// Do not expose DuckDB-only handlers from builds that intentionally omit bundled DuckDB. // Do not expose DuckDB-only handlers from builds that omit DuckDB sidecar support.
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
let api = let api =
api.route("/query/build-duckdb-attach-database-sql", post(routes::query::build_duckdb_attach_database_sql)); api.route("/query/build-duckdb-attach-database-sql", post(routes::query::build_duckdb_attach_database_sql));
+86 -21
View File
@@ -8,13 +8,13 @@ use dbx_core::agent_manager::{
}; };
use dbx_core::agent_service::{ use dbx_core::agent_service::{
build_agent_list, clear_agent_download_cache, fetch_registry, import_agent_driver, build_agent_list, clear_agent_download_cache, fetch_registry, import_agent_driver,
import_agents_from_zip as import_agents_from_zip_core, inspect_offline_zip, install_agent_driver, import_agents_from_package as import_agents_from_package_core, inspect_offline_package, install_agent_driver,
invalidate_registry_cache, reinstall_agent_jre, uninstall_agent_driver, uninstall_agent_jre, invalidate_registry_cache, reinstall_agent_jre, uninstall_agent_driver, uninstall_agent_jre,
upgrade_all_agent_drivers, AgentProgressEvent, OfflineImportPlan, upgrade_all_agent_drivers, AgentProgressEvent, OfflineImportPlan,
}; };
use dbx_core::driver_runtime::DriverRuntimeSummary; use dbx_core::driver_runtime::DriverRuntimeSummary;
use futures::Stream; use futures::Stream;
use serde::Deserialize; use serde::{Deserialize, Serialize};
use tokio::io::AsyncWriteExt; use tokio::io::AsyncWriteExt;
use tokio::sync::broadcast; use tokio::sync::broadcast;
@@ -41,6 +41,18 @@ pub struct AgentOperationRequest {
pub operation_id: Option<String>, pub operation_id: Option<String>,
} }
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AgentUpdateBlockersRequest {
pub db_types: Vec<String>,
}
#[derive(Debug, Clone, Serialize)]
pub struct AgentUpdateBlocker {
pub db_type: String,
pub label: String,
}
#[derive(Deserialize)] #[derive(Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct JavaRuntimeRequest { pub struct JavaRuntimeRequest {
@@ -138,10 +150,18 @@ pub async fn upgrade_all_agents(
Ok(Json(serde_json::to_value(result).map_err(|err| AppError::from(err.to_string()))?)) Ok(Json(serde_json::to_value(result).map_err(|err| AppError::from(err.to_string()))?))
} }
pub async fn check_agent_update_blockers(
State(state): State<Arc<WebState>>,
Json(req): Json<AgentUpdateBlockersRequest>,
) -> Result<Json<Vec<AgentUpdateBlocker>>, AppError> {
Ok(Json(agent_update_blockers(&state.app, &req.db_types).await))
}
pub async fn uninstall_agent( pub async fn uninstall_agent(
State(state): State<Arc<WebState>>, State(state): State<Arc<WebState>>,
Json(req): Json<AgentTypeRequest>, Json(req): Json<AgentTypeRequest>,
) -> Result<Json<serde_json::Value>, AppError> { ) -> Result<Json<serde_json::Value>, AppError> {
ensure_no_agent_update_blockers(&state.app, std::slice::from_ref(&req.db_type)).await.map_err(AppError::from)?;
uninstall_agent_driver(&state.app.agent_manager, &req.db_type).await.map_err(AppError::from)?; uninstall_agent_driver(&state.app.agent_manager, &req.db_type).await.map_err(AppError::from)?;
Ok(Json(serde_json::json!({ "ok": true }))) Ok(Json(serde_json::json!({ "ok": true })))
} }
@@ -200,14 +220,13 @@ pub async fn import_agents_from_zip(
} }
let file_name = field.file_name().unwrap_or("offline-drivers.zip").to_string(); let file_name = field.file_name().unwrap_or("offline-drivers.zip").to_string();
if !file_name.to_ascii_lowercase().ends_with(".zip") { let extension = offline_package_extension(&file_name)
return Err(AppError::from("Offline driver package must be a .zip file".to_string())); .ok_or_else(|| AppError::from("Offline driver package must be a .zip or .tar.zst file".to_string()))?;
} let package_path = tmp_dir.join(format!("agent-offline-{}.{}", uuid::Uuid::new_v4(), extension));
let zip_path = tmp_dir.join(format!("agent-offline-{}.zip", uuid::Uuid::new_v4()));
let tx = progress_sender(&state, "global").await; let tx = progress_sender(&state, "global").await;
let result = async { let result = async {
let mut upload = tokio::fs::File::create(&zip_path).await.map_err(|err| AppError::from(err.to_string()))?; let mut upload =
tokio::fs::File::create(&package_path).await.map_err(|err| AppError::from(err.to_string()))?;
let mut field = field; let mut field = field;
while let Some(chunk) = field.chunk().await.map_err(|err| AppError::from(err.to_string()))? { while let Some(chunk) = field.chunk().await.map_err(|err| AppError::from(err.to_string()))? {
upload.write_all(&chunk).await.map_err(|err| AppError::from(err.to_string()))?; upload.write_all(&chunk).await.map_err(|err| AppError::from(err.to_string()))?;
@@ -215,16 +234,16 @@ pub async fn import_agents_from_zip(
upload.flush().await.map_err(|err| AppError::from(err.to_string()))?; upload.flush().await.map_err(|err| AppError::from(err.to_string()))?;
drop(upload); drop(upload);
let plan = inspect_offline_zip(&zip_path).map_err(AppError::from)?; let plan = inspect_offline_package(&package_path).map_err(AppError::from)?;
ensure_no_offline_import_blockers(&state.app, &plan).await.map_err(AppError::from)?; ensure_no_offline_import_blockers(&state.app, &plan).await.map_err(AppError::from)?;
import_agents_from_zip_core(&state.app.agent_manager, &zip_path, |event| { import_agents_from_package_core(&state.app.agent_manager, &package_path, |event| {
send_progress_event(&tx, event.with_operation_id(&operation_id)) send_progress_event(&tx, event.with_operation_id(&operation_id))
}) })
.await .await
.map_err(AppError::from) .map_err(AppError::from)
} }
.await; .await;
let _ = std::fs::remove_file(&zip_path); let _ = std::fs::remove_file(&package_path);
let result = result?; let result = result?;
send_progress_event(&tx, AgentProgressEvent::step("done").with_operation_id(&operation_id)); send_progress_event(&tx, AgentProgressEvent::step("done").with_operation_id(&operation_id));
@@ -327,22 +346,46 @@ async fn ensure_no_agent_update_blockers(
state: &dbx_core::connection::AppState, state: &dbx_core::connection::AppState,
db_types: &[String], db_types: &[String],
) -> Result<(), String> { ) -> Result<(), String> {
let candidate_keys: std::collections::HashSet<&str> = db_types.iter().map(String::as_str).collect(); let blockers = update_blockers_from_keys(state.prepare_agent_driver_updates(db_types).await, db_types);
if candidate_keys.is_empty() { if blockers.is_empty() {
return Ok(()); return Ok(());
} }
let mut blockers = state let labels = blockers.into_iter().map(|blocker| blocker.label).collect::<Vec<_>>().join(", ");
.prepare_agent_driver_updates(db_types) Err(format!("Close these database connections before updating drivers: {labels}"))
.await }
async fn agent_update_blockers(state: &dbx_core::connection::AppState, db_types: &[String]) -> Vec<AgentUpdateBlocker> {
update_blockers_from_keys(state.active_agent_connection_driver_keys().await, db_types)
}
fn update_blockers_from_keys(
active_keys: std::collections::HashSet<String>,
db_types: &[String],
) -> Vec<AgentUpdateBlocker> {
let candidate_keys: std::collections::HashSet<&str> = db_types.iter().map(String::as_str).collect();
if candidate_keys.is_empty() {
return Vec::new();
}
let mut blockers = active_keys
.into_iter() .into_iter()
.filter(|key| candidate_keys.contains(key.as_str())) .filter(|key| candidate_keys.contains(key.as_str()))
.map(|key| dbx_core::agent_catalog::label_for_key(&key).unwrap_or(&key).to_string()) .map(|db_type| AgentUpdateBlocker {
label: dbx_core::agent_catalog::label_for_key(&db_type).unwrap_or(&db_type).to_string(),
db_type,
})
.collect::<Vec<_>>(); .collect::<Vec<_>>();
blockers.sort(); blockers.sort_by(|left, right| left.label.cmp(&right.label));
if blockers.is_empty() { blockers
Ok(()) }
fn offline_package_extension(file_name: &str) -> Option<&'static str> {
let lower_name = file_name.to_ascii_lowercase();
if lower_name.ends_with(".tar.zst") {
Some("tar.zst")
} else if lower_name.ends_with(".zip") {
Some("zip")
} else { } else {
Err(format!("Close these database connections before updating drivers: {}", blockers.join(", "))) None
} }
} }
@@ -361,3 +404,25 @@ async fn ensure_no_offline_import_blockers(
} }
ensure_no_agent_update_blockers(state, &driver_keys).await ensure_no_agent_update_blockers(state, &driver_keys).await
} }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn update_blockers_include_active_duckdb_connections() {
let active_keys = ["duckdb".to_string(), "oracle".to_string()].into_iter().collect();
let blockers = update_blockers_from_keys(active_keys, &["duckdb".to_string()]);
assert_eq!(blockers.len(), 1);
assert_eq!(blockers[0].db_type, "duckdb");
assert_eq!(blockers[0].label, "DuckDB");
}
#[test]
fn offline_package_extension_accepts_zip_and_tar_zstd() {
assert_eq!(offline_package_extension("dbx-agents.zip"), Some("zip"));
assert_eq!(offline_package_extension("dbx-agent-duckdb.TAR.ZST"), Some("tar.zst"));
assert_eq!(offline_package_extension("dbx-agent-duckdb.zst"), None);
}
}
+2 -2
View File
@@ -142,7 +142,7 @@ pub struct BuildCreateDatabaseSqlRequest {
#[derive(Deserialize)] #[derive(Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
pub struct BuildDuckDbAttachDatabaseSqlRequest { pub struct BuildDuckDbAttachDatabaseSqlRequest {
pub options: dbx_core::db_admin_sql::DuckDbAttachDatabaseSqlOptions, pub options: dbx_core::db_admin_sql::DuckDbAttachDatabaseSqlOptions,
} }
@@ -647,7 +647,7 @@ pub async fn build_create_database_sql(
dbx_core::db_admin_sql::build_create_database_sql(req.options).map(Json).map_err(AppError::from) dbx_core::db_admin_sql::build_create_database_sql(req.options).map(Json).map_err(AppError::from)
} }
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
pub async fn build_duckdb_attach_database_sql(Json(req): Json<BuildDuckDbAttachDatabaseSqlRequest>) -> Json<String> { pub async fn build_duckdb_attach_database_sql(Json(req): Json<BuildDuckDbAttachDatabaseSqlRequest>) -> Json<String> {
Json(dbx_core::db_admin_sql::build_duckdb_attach_database_sql(req.options)) Json(dbx_core::db_admin_sql::build_duckdb_attach_database_sql(req.options))
} }
+6 -6
View File
@@ -17,11 +17,11 @@ const i18n = {
bundles: "Offline Bundles", bundles: "Offline Bundles",
bundlesDesc: "Platform-specific ZIP packages that include the agent registry, database drivers, native agents, and the matching JRE.", bundlesDesc: "Platform-specific ZIP packages that include the agent registry, database drivers, native agents, and the matching JRE.",
drivers: "Database Drivers", drivers: "Database Drivers",
driversDesc: "Single-driver offline ZIPs for Java agents. Install the matching JRE separately when it is not already available.", driversDesc: "Single-driver .tar.zst packages for Java agents. Install the matching JRE separately when it is not already available.",
nativeAgents: "Native Agents", nativeAgents: "Native Agents",
nativeAgentsDesc: "Platform-specific offline ZIPs for Oracle, KingBase, and XuguDB. Import the ZIP directly in Driver Manager.", nativeAgentsDesc: "Platform-specific .tar.zst packages for DuckDB, Oracle, KingBase, and XuguDB. Import the package directly in Driver Manager.",
jre: "Java Runtime (JRE)", jre: "Java Runtime (JRE)",
jreDesc: "JRE packages used by agent-based database drivers. Required for Oracle, SQL Server, and other agent-managed connections.", jreDesc: "JRE packages used by Java agent-based database drivers such as SQL Server and Dameng.",
download: "Download", download: "Download",
installMethod: "Install", installMethod: "Install",
version: "Version", version: "Version",
@@ -53,11 +53,11 @@ const i18n = {
bundles: "整包下载", bundles: "整包下载",
bundlesDesc: "按平台提供的 ZIP 离线包,包含 Agent registry、数据库驱动、原生 Agent 和匹配的 JRE。", bundlesDesc: "按平台提供的 ZIP 离线包,包含 Agent registry、数据库驱动、原生 Agent 和匹配的 JRE。",
drivers: "数据库驱动", drivers: "数据库驱动",
driversDesc: "Java Agent 的单驱动离线 ZIP;目标机器尚未安装 JRE 时需要另外安装一次对应 JRE。", driversDesc: "Java Agent 的单驱动 .tar.zst 包;目标机器尚未安装 JRE 时需要另外安装一次对应 JRE。",
nativeAgents: "原生 Agent", nativeAgents: "原生 Agent",
nativeAgentsDesc: "Oracle、人大金仓和虚谷的按平台单驱动离线包,可直接在驱动管理中导入 ZIP。", nativeAgentsDesc: "DuckDB、Oracle、人大金仓和虚谷的按平台 .tar.zst 单驱动包,可直接在驱动管理中导入。",
jre: "Java 运行时 (JRE)", jre: "Java 运行时 (JRE)",
jreDesc: "Agent 驱动所需的 JRE 环境,Oracle、SQL Server 等数据库通过 Agent 连接时需要。", jreDesc: "Java Agent 驱动所需的 JRE 环境,例如 SQL Server、达梦等连接会使用。",
download: "下载", download: "下载",
installMethod: "安装方式", installMethod: "安装方式",
version: "版本", version: "版本",
+4 -4
View File
@@ -153,7 +153,7 @@ make
`make` 会使用锁文件安装依赖,然后启动 Tauri 桌面开发环境。第一次编译 Rust 依赖需要较长时间,后续启动会明显更快。 `make` 会使用锁文件安装依赖,然后启动 Tauri 桌面开发环境。第一次编译 Rust 依赖需要较长时间,后续启动会明显更快。
如果你修改的功能与 DuckDB 无关,推荐跳过 DuckDB 源码编译: 如需轻量桌面开发构建,可以保留 DuckDB sidecar 支持,同时跳过其他默认 Rust feature:
```bash ```bash
make dev-fast make dev-fast
@@ -172,10 +172,10 @@ pnpm install --frozen-lockfile
pnpm dev:tauri pnpm dev:tauri
``` ```
不需要 DuckDB 时使用: 使用相同的轻量构建:
```powershell ```powershell
pnpm tauri dev -- --no-default-features pnpm tauri dev -- --no-default-features --features duckdb-sidecar
``` ```
### 判断是否启动成功 ### 判断是否启动成功
@@ -451,7 +451,7 @@ git push --force-with-lease
### 第一次 Rust 编译很慢 ### 第一次 Rust 编译很慢
这是正常现象。与 DuckDB 无关的开发优先使用 `make dev-fast`、`make cargo-check-fast` 和 `make cargo-test-fast`。 这是正常现象。轻量桌面开发可以使用仍支持 DuckDB 的 `make dev-fast`。`make cargo-check-fast` 和 `make cargo-test-fast` 会跳过全部可选默认 feature,因此不会验证 DuckDB 专用路径。
### pnpm 提示版本不正确 ### pnpm 提示版本不正确
+4 -4
View File
@@ -153,7 +153,7 @@ make
`make` installs dependencies from the lockfile and starts the Tauri desktop development environment. The first Rust build can take a while; later starts are much faster. `make` installs dependencies from the lockfile and starts the Tauri desktop development environment. The first Rust build can take a while; later starts are much faster.
If your change does not involve DuckDB, skip the DuckDB source build: For a lightweight desktop build, keep DuckDB sidecar support while skipping the other default Rust features:
```bash ```bash
make dev-fast make dev-fast
@@ -172,10 +172,10 @@ pnpm install --frozen-lockfile
pnpm dev:tauri pnpm dev:tauri
``` ```
To skip DuckDB: For the same lightweight build:
```powershell ```powershell
pnpm tauri dev -- --no-default-features pnpm tauri dev -- --no-default-features --features duckdb-sidecar
``` ```
### Confirm the Environment Works ### Confirm the Environment Works
@@ -451,7 +451,7 @@ git push --force-with-lease
### The First Rust Build Is Slow ### The First Rust Build Is Slow
This is expected. Use `make dev-fast`, `make cargo-check-fast`, and `make cargo-test-fast` when DuckDB is not part of the change. This is expected. Use `make dev-fast` for a lightweight desktop build that still supports DuckDB. `make cargo-check-fast` and `make cargo-test-fast` skip all optional default features, so they do not validate DuckDB-specific paths.
### pnpm Reports the Wrong Version ### pnpm Reports the Wrong Version
+33 -13
View File
@@ -45,8 +45,9 @@ test("catalog falls back from R2 to CNB without using GitHub API", async () => {
access: { access: {
version: accessVersion, version: accessVersion,
jar: { jar: {
url: `https://github.com/t8y2/dbx/releases/download/agents-v0.2.64/dbx-agent-access-${accessVersion}.jar`, url: `https://github.com/t8y2/dbx/releases/download/agents-v0.2.64/dbx-agent-access-${accessVersion}.tar.zst`,
size: 1, size: 1,
format: "tar_zstd",
}, },
}, },
}, },
@@ -54,8 +55,9 @@ test("catalog falls back from R2 to CNB without using GitHub API", async () => {
"21": { "21": {
platforms: { platforms: {
"macos-aarch64": { "macos-aarch64": {
url: "https://github.com/t8y2/dbx/releases/download/agents-v0.2.64/dbx-jre-21-macos-aarch64.tar.gz", url: "https://github.com/t8y2/dbx/releases/download/agents-v0.2.64/dbx-jre-21-macos-aarch64.tar.zst",
size: 1, size: 1,
format: "tar_zstd",
}, },
}, },
}, },
@@ -77,7 +79,7 @@ test("unknown fallback asset sizes render as unavailable", () => {
assert.equal(formatSize(0), "—"); assert.equal(formatSize(0), "—");
}); });
test("Java agent ZIPs are preferred over raw JARs", () => { test("Java agent tar.zst packages are preferred over raw JARs", () => {
const accessVersion = driverVersions.access; const accessVersion = driverVersions.access;
const entries = buildDriverEntries([ const entries = buildDriverEntries([
{ {
@@ -86,17 +88,17 @@ test("Java agent ZIPs are preferred over raw JARs", () => {
size: 1024, size: 1024,
}, },
{ {
name: `dbx-agent-access-${accessVersion}.zip`, name: `dbx-agent-access-${accessVersion}.tar.zst`,
browser_download_url: `https://example.com/dbx-agent-access-${accessVersion}.zip`, browser_download_url: `https://example.com/dbx-agent-access-${accessVersion}.tar.zst`,
size: 2048, size: 2048,
}, },
]); ]);
assert.equal(entries[0]?.key, "access"); assert.equal(entries[0]?.key, "access");
assert.equal(entries[0]?.jar.url, `https://example.com/dbx-agent-access-${accessVersion}.zip`); assert.equal(entries[0]?.jar.url, `https://example.com/dbx-agent-access-${accessVersion}.tar.zst`);
}); });
test("KingBase native ZIPs are preferred over raw release executables", () => { test("KingBase native tar.zst packages are preferred over raw release executables", () => {
const entries = buildNativeAgentEntries([ const entries = buildNativeAgentEntries([
{ {
name: "dbx-agent-kingbase-windows-x64.exe", name: "dbx-agent-kingbase-windows-x64.exe",
@@ -109,13 +111,13 @@ test("KingBase native ZIPs are preferred over raw release executables", () => {
size: 2048, size: 2048,
}, },
{ {
name: "dbx-agent-kingbase-0.1.34-windows-x64.zip", name: "dbx-agent-kingbase-0.1.34-windows-x64.tar.zst",
browser_download_url: "https://example.com/dbx-agent-kingbase-0.1.34-windows-x64.zip", browser_download_url: "https://example.com/dbx-agent-kingbase-0.1.34-windows-x64.tar.zst",
size: 4096, size: 4096,
}, },
{ {
name: "dbx-agent-kingbase-0.1.34-linux-x64.zip", name: "dbx-agent-kingbase-0.1.34-linux-x64.tar.zst",
browser_download_url: "https://example.com/dbx-agent-kingbase-0.1.34-linux-x64.zip", browser_download_url: "https://example.com/dbx-agent-kingbase-0.1.34-linux-x64.tar.zst",
size: 3072, size: 3072,
}, },
]); ]);
@@ -127,14 +129,32 @@ test("KingBase native ZIPs are preferred over raw release executables", () => {
key: "kingbase", key: "kingbase",
version: "0.1.34", version: "0.1.34",
platformKey: "linux-x64", platformKey: "linux-x64",
filename: "dbx-agent-kingbase-0.1.34-linux-x64.zip", filename: "dbx-agent-kingbase-0.1.34-linux-x64.tar.zst",
}, },
{ {
key: "kingbase", key: "kingbase",
version: "0.1.34", version: "0.1.34",
platformKey: "windows-x64", platformKey: "windows-x64",
filename: "dbx-agent-kingbase-0.1.34-windows-x64.zip", filename: "dbx-agent-kingbase-0.1.34-windows-x64.tar.zst",
}, },
], ],
); );
}); });
test("DuckDB native tar.zst packages appear in the native catalog", () => {
const entries = buildNativeAgentEntries([
{
name: "dbx-agent-duckdb-0.1.0-macos-aarch64.tar.zst",
browser_download_url: "https://example.com/dbx-agent-duckdb-0.1.0-macos-aarch64.tar.zst",
size: 4096,
},
]);
assert.deepEqual(entries.map(({ key, platformKey, filename }) => ({ key, platformKey, filename })), [
{
key: "duckdb",
platformKey: "macos-aarch64",
filename: "dbx-agent-duckdb-0.1.0-macos-aarch64.tar.zst",
},
]);
});
+11 -26
View File
@@ -21,6 +21,7 @@ export interface GitHubReleaseAsset {
interface AgentRegistryArtifact { interface AgentRegistryArtifact {
url: string; url: string;
size: number; size: number;
format?: "tar_zstd";
} }
interface AgentRegistryDriver { interface AgentRegistryDriver {
@@ -88,7 +89,7 @@ const GITHUB_RELEASE_DOWNLOAD_PREFIX = "https://github.com/t8y2/dbx/releases/dow
const CNB_RELEASE_DOWNLOAD_PREFIX = "https://cnb.cool/dbxio.com/dbx/-/releases/download/"; const CNB_RELEASE_DOWNLOAD_PREFIX = "https://cnb.cool/dbxio.com/dbx/-/releases/download/";
const MIN_APP_VERSION = "0.6.0"; const MIN_APP_VERSION = "0.6.0";
const driverVersionMap = driverVersions as Record<string, string>; const driverVersionMap = driverVersions as Record<string, string>;
const nativeDriverKeys = new Set(["oracle", "kingbase", "xugu"]); const nativeDriverKeys = new Set(["duckdb", "oracle", "kingbase", "xugu"]);
const platformLabels: Record<string, string> = { const platformLabels: Record<string, string> = {
"macos-aarch64": "macOS (Apple Silicon)", "macos-aarch64": "macOS (Apple Silicon)",
@@ -107,6 +108,7 @@ const driverLabels: Record<string, string> = {
databend: "Databend", databend: "Databend",
databricks: "Databricks", databricks: "Databricks",
db2: "DB2", db2: "DB2",
duckdb: "DuckDB",
etcd: "etcd", etcd: "etcd",
exasol: "Exasol", exasol: "Exasol",
firebird: "Firebird", firebird: "Firebird",
@@ -182,31 +184,14 @@ function registryReleaseAssets(registry: AgentRegistry): GitHubReleaseAsset[] {
for (const [jreKey, jre] of Object.entries(registry.jres ?? {})) { for (const [jreKey, jre] of Object.entries(registry.jres ?? {})) {
for (const [platformKey, artifact] of Object.entries(jre.platforms ?? {})) { for (const [platformKey, artifact] of Object.entries(jre.platforms ?? {})) {
addArtifact(artifact, `dbx-jre-${jreKey}-${platformKey}.tar.gz`); addArtifact(artifact, `dbx-jre-${jreKey}-${platformKey}.tar.zst`);
} }
} }
for (const [driverKey, driver] of Object.entries(registry.drivers ?? {})) { for (const [driverKey, driver] of Object.entries(registry.drivers ?? {})) {
addArtifact(driver.jar, `dbx-agent-${driverKey}.jar`); addArtifact(driver.jar, `dbx-agent-${driverKey}-${driver.version ?? "latest"}.tar.zst`);
if (driver.version && driver.jar?.url && driver.jar.size > 0) {
const packageName = `dbx-agent-${driverKey}-${driver.version}.zip`;
assets.set(packageName, {
name: packageName,
size: 0,
browser_download_url: siblingReleaseAssetUrl(driver.jar.url, packageName),
});
}
for (const [platformKey, artifact] of Object.entries(driver.native ?? {})) { for (const [platformKey, artifact] of Object.entries(driver.native ?? {})) {
addArtifact(artifact, `dbx-agent-${driverKey}-${platformKey}`); addArtifact(artifact, `dbx-agent-${driverKey}-${driver.version ?? "latest"}-${platformKey}.tar.zst`);
const filename = releaseAssetName(artifact.url, "");
if (driver.version && filename.includes(`-${driver.version}-${platformKey}`)) {
const packageName = `${filename.replace(/\.exe$/, "")}.zip`;
assets.set(packageName, {
name: packageName,
size: 0,
browser_download_url: siblingReleaseAssetUrl(artifact.url, packageName),
});
}
} }
} }
@@ -263,7 +248,7 @@ export function buildJdbcPluginDownloadEntry(): JdbcPluginDownloadEntry {
export function buildJreEntries(assets: GitHubReleaseAsset[]): JreDisplayEntry[] { export function buildJreEntries(assets: GitHubReleaseAsset[]): JreDisplayEntry[] {
return assets return assets
.map((asset) => { .map((asset) => {
const match = /^dbx-jre-(\d+)-(.+)\.tar\.gz$/.exec(asset.name); const match = /^dbx-jre-(\d+)-(.+)\.tar\.(?:zst|gz)$/.exec(asset.name);
if (!match) return null; if (!match) return null;
const [, jreKey, platformKey] = match; const [, jreKey, platformKey] = match;
@@ -287,7 +272,7 @@ export function buildDriverEntries(assets: GitHubReleaseAsset[]): DriverDisplayE
return currentJavaDriverKeys return currentJavaDriverKeys
.map((key) => { .map((key) => {
const version = driverVersionMap[key] ?? ""; const version = driverVersionMap[key] ?? "";
const asset = byName.get(`dbx-agent-${key}-${version}.zip`) ?? byName.get(`dbx-agent-${key}-${version}.jar`) ?? byName.get(`dbx-agent-${key}.jar`); const asset = byName.get(`dbx-agent-${key}-${version}.tar.zst`) ?? byName.get(`dbx-agent-${key}-${version}.zip`) ?? byName.get(`dbx-agent-${key}-${version}.jar`) ?? byName.get(`dbx-agent-${key}.jar`);
if (!asset) return null; if (!asset) return null;
return { return {
@@ -307,9 +292,9 @@ export function buildNativeAgentEntries(assets: GitHubReleaseAsset[]): NativeAge
const platforms = "macos-aarch64|macos-x64|linux-aarch64|linux-x64|windows-aarch64|windows-x64"; const platforms = "macos-aarch64|macos-x64|linux-aarch64|linux-x64|windows-aarch64|windows-x64";
for (const asset of assets) { for (const asset of assets) {
const packageMatch = new RegExp(`^dbx-agent-(oracle|kingbase|xugu)-(.+)-(${platforms})\\.zip$`).exec(asset.name); const packageMatch = new RegExp(`^dbx-agent-(duckdb|oracle|kingbase|xugu)-(.+)-(${platforms})\\.(?:tar\\.zst|zip)$`).exec(asset.name);
const versionedMatch = new RegExp(`^dbx-agent-(oracle|kingbase|xugu)-(.+)-(${platforms})(?:\\.exe)?$`).exec(asset.name); const versionedMatch = new RegExp(`^dbx-agent-(duckdb|oracle|kingbase|xugu)-(.+)-(${platforms})(?:\\.exe)?$`).exec(asset.name);
const legacyMatch = new RegExp(`^dbx-agent-(oracle|kingbase|xugu)-(${platforms})(?:\\.exe)?$`).exec(asset.name); const legacyMatch = new RegExp(`^dbx-agent-(duckdb|oracle|kingbase|xugu)-(${platforms})(?:\\.exe)?$`).exec(asset.name);
const match = packageMatch ?? versionedMatch ?? legacyMatch; const match = packageMatch ?? versionedMatch ?? legacyMatch;
if (!match) continue; if (!match) continue;
+2 -3
View File
@@ -355,7 +355,7 @@ async fn detect_pulsar_version(client: &reqwest::Client, base_url: &str) -> Puls
### 启用 MQ 功能 ### 启用 MQ 功能
```bash ```bash
# 默认已启用(default = ["duckdb-bundled", "mq-admin"]) # 默认已启用(default 包含 "duckdb-sidecar" 和 "mq-admin")
cargo build --release cargo build --release
# 或显式指定 # 或显式指定
@@ -364,7 +364,7 @@ cargo build --release --features mq-admin
### 禁用 MQ 功能 ### 禁用 MQ 功能
```bash ```bash
cargo build --release --no-default-features --features duckdb-bundled cargo build -p dbx --release --no-default-features --features duckdb-sidecar,sqlite-sqlcipher,system-fonts
``` ```
### 检查编译 ### 检查编译
@@ -507,4 +507,3 @@ const response = await mqRawRequest(connectionId, {
console.log(response.body) console.log(response.body)
``` ```
+2 -3
View File
@@ -13,8 +13,8 @@ name = "dbx_lib"
crate-type = ["staticlib", "cdylib", "rlib"] crate-type = ["staticlib", "cdylib", "rlib"]
[features] [features]
default = ["duckdb-bundled", "mq-admin", "sqlite-sqlcipher", "system-fonts"] default = ["duckdb-sidecar", "mq-admin", "sqlite-sqlcipher", "system-fonts"]
duckdb-bundled = ["duckdb", "duckdb/parquet", "dbx-core/duckdb-bundled"] duckdb-sidecar = ["dbx-core/duckdb-sidecar"]
mq-admin = ["dbx-core/mq-admin"] mq-admin = ["dbx-core/mq-admin"]
sqlite-sqlcipher = ["dbx-core/sqlite-sqlcipher"] sqlite-sqlcipher = ["dbx-core/sqlite-sqlcipher"]
system-fonts = ["dbx-core/system-fonts"] system-fonts = ["dbx-core/system-fonts"]
@@ -47,7 +47,6 @@ tauri-plugin-window-state = "2"
redis = { version = "0.32.2", features = ["tokio-comp", "tls-rustls", "tokio-rustls-comp", "cluster-async"] } redis = { version = "0.32.2", features = ["tokio-comp", "tls-rustls", "tokio-rustls-comp", "cluster-async"] }
rustls = { version = "0.23", features = ["aws-lc-rs"] } rustls = { version = "0.23", features = ["aws-lc-rs"] }
portpicker = "0.1.1" portpicker = "0.1.1"
duckdb = { version = "1.3.2", optional = true }
tiberius = { version = "0.12.3", default-features = false, features = ["tds73", "chrono", "rust_decimal", "rustls", "sql-browser-tokio"] } tiberius = { version = "0.12.3", default-features = false, features = ["tds73", "chrono", "rust_decimal", "rustls", "sql-browser-tokio"] }
tokio-util = { version = "0.7", features = ["compat"] } tokio-util = { version = "0.7", features = ["compat"] }
reqwest = { version = "0.12", features = ["json", "stream"] } reqwest = { version = "0.12", features = ["json", "stream"] }
+8 -6
View File
@@ -5,7 +5,7 @@ use tauri::{Emitter, State};
use dbx_core::agent_manager::{AgentDriverInfo, DriverStoreUsage, JavaRuntimeConfig, JavaRuntimeMode, DEFAULT_JRE_KEY}; use dbx_core::agent_manager::{AgentDriverInfo, DriverStoreUsage, JavaRuntimeConfig, JavaRuntimeMode, DEFAULT_JRE_KEY};
use dbx_core::agent_service::{ use dbx_core::agent_service::{
build_agent_list, clear_agent_download_cache, fetch_registry_from, import_agent_driver, build_agent_list, clear_agent_download_cache, fetch_registry_from, import_agent_driver,
import_agents_from_zip as import_agents_from_zip_core, inspect_offline_zip, install_agent_driver_from, import_agents_from_package as import_agents_from_package_core, inspect_offline_package, install_agent_driver_from,
invalidate_registry_cache, reinstall_agent_jre_from, uninstall_agent_driver, uninstall_agent_jre, invalidate_registry_cache, reinstall_agent_jre_from, uninstall_agent_driver, uninstall_agent_jre,
upgrade_all_agent_drivers_from, AgentProgressEvent, OfflineImportPlan, UpgradeAllAgentDriversResult, upgrade_all_agent_drivers_from, AgentProgressEvent, OfflineImportPlan, UpgradeAllAgentDriversResult,
}; };
@@ -111,6 +111,7 @@ pub async fn check_agent_update_blockers(
#[tauri::command] #[tauri::command]
pub async fn uninstall_agent(state: State<'_, Arc<AppState>>, db_type: String) -> Result<(), String> { pub async fn uninstall_agent(state: State<'_, Arc<AppState>>, db_type: String) -> Result<(), String> {
ensure_no_agent_update_blockers(state.inner().as_ref(), std::slice::from_ref(&db_type)).await?;
uninstall_agent_driver(&state.agent_manager, &db_type).await uninstall_agent_driver(&state.agent_manager, &db_type).await
} }
@@ -166,14 +167,15 @@ pub async fn import_agents_from_zip(
operation_id: Option<String>, operation_id: Option<String>,
) -> Result<u32, String> { ) -> Result<u32, String> {
let am = &state.agent_manager; let am = &state.agent_manager;
let zip_path = std::path::PathBuf::from(&path); let package_path = std::path::PathBuf::from(&path);
let plan = inspect_offline_zip(&zip_path)?; let plan = inspect_offline_package(&package_path)?;
ensure_no_offline_import_blockers(state.inner().as_ref(), &plan).await?; ensure_no_offline_import_blockers(state.inner().as_ref(), &plan).await?;
let app_handle = app.clone(); let app_handle = app.clone();
let operation_id = operation_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); let operation_id = operation_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
let result = let result = import_agents_from_package_core(am, &package_path, |event| {
import_agents_from_zip_core(am, &zip_path, |event| emit_agent_progress(&app_handle, &operation_id, event)) emit_agent_progress(&app_handle, &operation_id, event)
.await?; })
.await?;
let count = result.drivers_installed.len() as u32; let count = result.drivers_installed.len() as u32;
emit_agent_progress(&app, &operation_id, AgentProgressEvent::step("done")); emit_agent_progress(&app, &operation_id, AgentProgressEvent::step("done"));
Ok(count) Ok(count)
+7 -19
View File
@@ -839,13 +839,13 @@ async fn test_connection_with_info_inner(
// are reset after both successful and failed Redis tests. // are reset after both successful and failed Redis tests.
test_redis_connection(state, &tunnel_id, &config, &host, port, connect_timeout).await test_redis_connection(state, &tunnel_id, &config, &host, port, connect_timeout).await
} }
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
DatabaseType::DuckDb => { DatabaseType::DuckDb => {
state.test_duckdb_connection_config(&config).await?; state.test_duckdb_connection_config(&config).await?;
Ok("Connection successful".to_string()) Ok("Connection successful".to_string())
} }
#[cfg(not(feature = "duckdb-bundled"))] #[cfg(not(feature = "duckdb-sidecar"))]
DatabaseType::DuckDb => Err("DuckDB support not compiled (enable duckdb-bundled feature)".to_string()), DatabaseType::DuckDb => Err("DuckDB support is not compiled in this build".to_string()),
DatabaseType::MongoDb => { DatabaseType::MongoDb => {
if mongo_uses_legacy_driver(&config) { if mongo_uses_legacy_driver(&config) {
let am = &state.agent_manager; let am = &state.agent_manager;
@@ -1144,22 +1144,10 @@ pub async fn connect_db(
}; };
con con
} }
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
DatabaseType::DuckDb => { DatabaseType::DuckDb => state.create_duckdb_pool(&db_config).await?,
let con = db::duckdb_driver::connect_path(&expand_tilde(&db_config.host))?; #[cfg(not(feature = "duckdb-sidecar"))]
{ DatabaseType::DuckDb => return Err("DuckDB support is not compiled in this build".to_string()),
let locked = con.lock().map_err(|e| e.to_string())?;
for attached in &db_config.attached_databases {
dbx_core::schema::duckdb_attach_database(&locked, &attached.name, &expand_tilde(&attached.path))?;
}
if let Some(script) = db_config.init_script.as_deref() {
db::duckdb_driver::run_init_script(&locked, script)?;
}
}
PoolKind::DuckDb(con)
}
#[cfg(not(feature = "duckdb-bundled"))]
DatabaseType::DuckDb => return Err("DuckDB support not compiled (enable duckdb-bundled feature)".to_string()),
DatabaseType::MongoDb => { DatabaseType::MongoDb => {
if mongo_uses_legacy_driver(&db_config) { if mongo_uses_legacy_driver(&db_config) {
let mut client = let mut client =
+1 -1
View File
@@ -428,7 +428,7 @@ pub fn build_create_database_sql(options: dbx_core::db_admin_sql::CreateDatabase
dbx_core::db_admin_sql::build_create_database_sql(options) dbx_core::db_admin_sql::build_create_database_sql(options)
} }
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
#[tauri::command] #[tauri::command]
pub fn build_duckdb_attach_database_sql( pub fn build_duckdb_attach_database_sql(
options: dbx_core::db_admin_sql::DuckDbAttachDatabaseSqlOptions, options: dbx_core::db_admin_sql::DuckDbAttachDatabaseSqlOptions,
+1 -1
View File
@@ -1498,7 +1498,7 @@ pub fn run() {
commands::query::build_search_result_where, commands::query::build_search_result_where,
commands::query::build_rename_object_sql, commands::query::build_rename_object_sql,
commands::query::build_create_database_sql, commands::query::build_create_database_sql,
#[cfg(feature = "duckdb-bundled")] #[cfg(feature = "duckdb-sidecar")]
commands::query::build_duckdb_attach_database_sql, commands::query::build_duckdb_attach_database_sql,
commands::query::build_sqlite_attach_database_sql, commands::query::build_sqlite_attach_database_sql,
commands::query::build_drop_object_sql, commands::query::build_drop_object_sql,
-10
View File
@@ -2,15 +2,5 @@
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] #![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
fn main() { fn main() {
#[cfg(feature = "duckdb-bundled")]
if std::env::args().any(|arg| arg == "--duckdb-worker") {
let runtime = tokio::runtime::Runtime::new().expect("Failed to create DuckDB worker runtime");
if let Err(err) = runtime.block_on(dbx_core::db::duckdb_worker_runtime::run_stdio_worker()) {
eprintln!("DuckDB worker failed: {err}");
std::process::exit(1);
}
return;
}
dbx_lib::run(); dbx_lib::run();
} }