fix(duckdb): statically link Windows worker runtime

Closes #5313
This commit is contained in:
t8y2
2026-08-04 15:08:04 +08:00
parent 947cc16d92
commit cd5c222f95
7 changed files with 289 additions and 4 deletions
@@ -84,6 +84,18 @@ test("builds a manually versioned module even without runtime file changes", ()
assert.equal(result.versions.duckdb, "0.1.1");
});
test("bumps DuckDB when its Cargo target configuration changes", () => {
const result = evaluateAgentVersionBump({
versions: { duckdb: "0.1.2" },
changedFiles: ["agents/drivers/duckdb/.cargo/config.toml"],
moduleExists: (path) => path === "agents/drivers/duckdb",
readModuleFile: () => "",
});
assert.equal(result.versions.duckdb, "0.1.3");
assert.deepEqual(result.nativeModules, ["duckdb"]);
});
test("builds only common-dependent Java modules for a shared runtime change", () => {
const existing = new Set([
"agents/drivers/access",
+13 -3
View File
@@ -447,6 +447,10 @@ jobs:
with:
toolchain: nightly-2026-07-22
components: rust-src
- uses: actions/setup-python@v5
if: runner.os == 'Windows'
with:
python-version: "3.13"
- uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
with:
version: "v0.10.0"
@@ -460,14 +464,14 @@ jobs:
-e CARGO_INCREMENTAL=0 \
-e CARGO_TARGET_DIR=/workspace/target/duckdb-driver \
-v "${{ github.workspace }}:/workspace" \
-w /workspace \
-w /workspace/agents/drivers/duckdb \
"${{ matrix.manylinux_image }}" \
bash -lc '
mkdir -p "$HOME"
curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.97.1
source "$HOME/.cargo/env"
cargo build \
--manifest-path agents/drivers/duckdb/Cargo.toml \
--manifest-path Cargo.toml \
--locked \
--release \
--bin dbx-duckdb-driver \
@@ -478,18 +482,24 @@ jobs:
if [ "${{ matrix.win7 }}" = "true" ]; then
BUILD_STD=(-Z build-std=std,panic_abort)
fi
pushd agents/drivers/duckdb
cargo build \
--manifest-path agents/drivers/duckdb/Cargo.toml \
--manifest-path Cargo.toml \
--locked \
--release \
--bin dbx-duckdb-driver \
--target "${{ matrix.target }}" \
"${BUILD_STD[@]}"
popd
fi
mkdir -p release-native
cp \
"target/duckdb-driver/${{ matrix.target }}/release/dbx-duckdb-driver${{ matrix.extension }}" \
"release-native/dbx-agent-duckdb-${{ matrix.platform }}${{ matrix.extension }}"
if [ "${{ runner.os }}" = "Windows" ]; then
python agents/scripts/validate_windows_pe_dependencies.py \
"release-native/dbx-agent-duckdb-${{ matrix.platform }}${{ matrix.extension }}"
fi
if [ -n "${{ matrix.manylinux_image }}" ]; then
docker run --rm \
--user "$(id -u):$(id -g)" \
+51
View File
@@ -148,6 +148,51 @@ jobs:
}
Get-FileHash -LiteralPath $installer.FullName -Algorithm SHA256
duckdb-windows-driver:
needs: changes
if: needs.changes.outputs.duckdb_windows == 'true'
runs-on: windows-2022
timeout-minutes: 60
env:
CARGO_INCREMENTAL: "0"
CARGO_TARGET_DIR: ${{ github.workspace }}/target/duckdb-driver
RUSTC_WRAPPER: sccache
steps:
- uses: actions/checkout@v5
- name: Setup Rust for Windows 7
uses: dtolnay/rust-toolchain@nightly
with:
toolchain: nightly-2026-07-22
components: rust-src
- uses: actions/setup-python@v5
with:
python-version: "3.13"
- uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
with:
version: "v0.10.0"
- name: Build DuckDB Windows driver
shell: bash
run: |
pushd agents/drivers/duckdb
cargo build \
--locked \
--release \
--bin dbx-duckdb-driver \
--target x86_64-win7-windows-msvc \
-Z build-std=std,panic_abort
popd
- name: Validate DuckDB Windows driver
shell: bash
run: |
DRIVER="target/duckdb-driver/x86_64-win7-windows-msvc/release/dbx-duckdb-driver.exe"
python agents/scripts/validate_windows_pe_dependencies.py "$DRIVER"
"$DRIVER" < /dev/null
rust-fmt-clippy:
needs: changes
if: needs.changes.outputs.rust == 'true'
@@ -370,6 +415,7 @@ jobs:
rust_full: ${{ steps.rust-mode.outputs.full }}
jdbc: ${{ steps.filter.outputs.jdbc }}
agents: ${{ steps.filter.outputs.agents }}
duckdb_windows: ${{ steps.filter.outputs.duckdb_windows }}
nix: ${{ steps.filter.outputs.nix }}
windows_win7_bundle: ${{ steps.filter.outputs.windows_win7_bundle }}
github_scripts: ${{ steps.filter.outputs.github_scripts }}
@@ -421,6 +467,11 @@ jobs:
- '.github/scripts/bump-agent-versions.test.mjs'
- '.github/workflows/agents-release.yml'
- '.github/workflows/ci.yml'
duckdb_windows:
- 'agents/drivers/duckdb/**'
- 'agents/scripts/validate_windows_pe_dependencies.py'
- '.github/workflows/agents-release.yml'
- '.github/workflows/ci.yml'
nix:
# These advisory checks validate the pnpm and Cargo dependency closures.
- 'package.json'
+2
View File
@@ -0,0 +1,2 @@
[target.'cfg(all(windows, target_env = "msvc"))']
rustflags = ["-C", "target-feature=+crt-static"]
+5 -1
View File
@@ -7,7 +7,8 @@ existing newline-delimited JSON worker runtime, while keeping DuckDB and
## Build
```bash
cargo build --manifest-path agents/drivers/duckdb/Cargo.toml --release --bin dbx-duckdb-driver
cd agents/drivers/duckdb
cargo build --release --bin dbx-duckdb-driver
```
Point DBX at the resulting executable with:
@@ -35,6 +36,9 @@ DBX decompresses the package itself, so users do not need to install `zstd`,
DuckDB, or a separate database driver. The existing aggregate offline `.zip`
packages remain supported for backward compatibility.
Windows MSVC artifacts statically link the Visual C++ runtime so they also run
on fresh Windows installations without a separate redistributable package.
## Current scope
The driver implements connect, execute, database/schema/table/column metadata,
@@ -0,0 +1,130 @@
#!/usr/bin/env python3
import argparse
import struct
from pathlib import Path
class PeFormatError(ValueError):
pass
def _read_u16(data: bytes, offset: int) -> int:
if offset < 0 or offset + 2 > len(data):
raise PeFormatError("unexpected end of PE file")
return struct.unpack_from("<H", data, offset)[0]
def _read_u32(data: bytes, offset: int) -> int:
if offset < 0 or offset + 4 > len(data):
raise PeFormatError("unexpected end of PE file")
return struct.unpack_from("<I", data, offset)[0]
def _read_c_string(data: bytes, offset: int) -> str:
if offset < 0 or offset >= len(data):
raise PeFormatError("PE string offset is outside the file")
end = data.find(b"\0", offset)
if end < 0:
raise PeFormatError("unterminated PE string")
try:
return data[offset:end].decode("ascii")
except UnicodeDecodeError as error:
raise PeFormatError("PE import name is not ASCII") from error
def imported_dlls(path: Path) -> list[str]:
data = path.read_bytes()
if len(data) < 64 or data[:2] != b"MZ":
raise PeFormatError("missing DOS header")
pe_offset = _read_u32(data, 0x3C)
if data[pe_offset : pe_offset + 4] != b"PE\0\0":
raise PeFormatError("missing PE signature")
section_count = _read_u16(data, pe_offset + 6)
optional_header_size = _read_u16(data, pe_offset + 20)
optional_header_offset = pe_offset + 24
optional_magic = _read_u16(data, optional_header_offset)
if optional_magic == 0x20B:
data_directories_offset = optional_header_offset + 112
elif optional_magic == 0x10B:
data_directories_offset = optional_header_offset + 96
else:
raise PeFormatError(f"unsupported PE optional header magic: 0x{optional_magic:04x}")
import_directory_rva = _read_u32(data, data_directories_offset + 8)
import_directory_size = _read_u32(data, data_directories_offset + 12)
if import_directory_rva == 0 or import_directory_size == 0:
return []
size_of_headers = _read_u32(data, optional_header_offset + 60)
section_table_offset = optional_header_offset + optional_header_size
sections = []
for index in range(section_count):
section_offset = section_table_offset + index * 40
sections.append(
(
_read_u32(data, section_offset + 12),
_read_u32(data, section_offset + 8),
_read_u32(data, section_offset + 20),
_read_u32(data, section_offset + 16),
)
)
def rva_to_offset(rva: int) -> int:
if rva < size_of_headers:
return rva
for virtual_address, virtual_size, raw_offset, raw_size in sections:
span = max(virtual_size, raw_size)
if virtual_address <= rva < virtual_address + span:
delta = rva - virtual_address
if delta >= raw_size:
break
return raw_offset + delta
raise PeFormatError(f"PE RVA 0x{rva:x} is not backed by file data")
descriptor_offset = rva_to_offset(import_directory_rva)
imports = []
for _ in range(4096):
if descriptor_offset + 20 > len(data):
raise PeFormatError("truncated PE import descriptor")
descriptor = data[descriptor_offset : descriptor_offset + 20]
if descriptor == b"\0" * 20:
return sorted(set(imports), key=str.casefold)
name_rva = _read_u32(data, descriptor_offset + 12)
if name_rva == 0:
raise PeFormatError("PE import descriptor has no DLL name")
imports.append(_read_c_string(data, rva_to_offset(name_rva)))
descriptor_offset += 20
raise PeFormatError("PE import descriptor table is not terminated")
def forbidden_msvc_runtime_dlls(imports: list[str]) -> list[str]:
return sorted(
{name for name in imports if name.casefold().startswith(("msvcp", "vcruntime"))},
key=str.casefold,
)
def main() -> int:
parser = argparse.ArgumentParser(description="Reject Windows PE files that require the Visual C++ runtime")
parser.add_argument("binary", type=Path)
args = parser.parse_args()
try:
imports = imported_dlls(args.binary)
except (OSError, PeFormatError) as error:
parser.error(str(error))
forbidden = forbidden_msvc_runtime_dlls(imports)
if forbidden:
parser.error(f"dynamic Visual C++ runtime dependencies found: {', '.join(forbidden)}")
print(f"Validated {args.binary}: {len(imports)} imported DLLs, no dynamic Visual C++ runtime")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,76 @@
import struct
import tempfile
import unittest
from pathlib import Path
from validate_windows_pe_dependencies import PeFormatError, forbidden_msvc_runtime_dlls, imported_dlls
def build_test_pe(imports: list[str]) -> bytes:
data = bytearray(4096)
pe_offset = 0x80
optional_header_offset = pe_offset + 24
optional_header_size = 0xF0
section_table_offset = optional_header_offset + optional_header_size
section_rva = 0x1000
section_offset = 0x200
import_directory_rva = section_rva
data[:2] = b"MZ"
struct.pack_into("<I", data, 0x3C, pe_offset)
data[pe_offset : pe_offset + 4] = b"PE\0\0"
struct.pack_into("<H", data, pe_offset + 4, 0x8664)
struct.pack_into("<H", data, pe_offset + 6, 1)
struct.pack_into("<H", data, pe_offset + 20, optional_header_size)
struct.pack_into("<H", data, optional_header_offset, 0x20B)
struct.pack_into("<I", data, optional_header_offset + 60, section_offset)
struct.pack_into("<I", data, optional_header_offset + 112 + 8, import_directory_rva)
struct.pack_into("<I", data, optional_header_offset + 112 + 12, (len(imports) + 1) * 20)
data[section_table_offset : section_table_offset + 8] = b".rdata\0\0"
struct.pack_into("<I", data, section_table_offset + 8, 0x800)
struct.pack_into("<I", data, section_table_offset + 12, section_rva)
struct.pack_into("<I", data, section_table_offset + 16, 0x800)
struct.pack_into("<I", data, section_table_offset + 20, section_offset)
name_offset = section_offset + 0x200
for index, name in enumerate(imports):
descriptor_offset = section_offset + index * 20
name_rva = section_rva + (name_offset - section_offset)
struct.pack_into("<I", data, descriptor_offset + 12, name_rva)
encoded = name.encode("ascii") + b"\0"
data[name_offset : name_offset + len(encoded)] = encoded
name_offset += len(encoded)
return bytes(data)
class WindowsPeDependenciesTest(unittest.TestCase):
def test_reads_imported_dll_names(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
path = Path(temp_dir) / "driver.exe"
path.write_bytes(build_test_pe(["KERNEL32.dll", "bcrypt.dll"]))
self.assertEqual(imported_dlls(path), ["bcrypt.dll", "KERNEL32.dll"])
def test_rejects_dynamic_visual_cpp_runtime(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
path = Path(temp_dir) / "driver.exe"
path.write_bytes(build_test_pe(["KERNEL32.dll", "MSVCP140.dll", "VCRUNTIME140_1.dll"]))
self.assertEqual(
forbidden_msvc_runtime_dlls(imported_dlls(path)),
["MSVCP140.dll", "VCRUNTIME140_1.dll"],
)
def test_rejects_non_pe_files(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
path = Path(temp_dir) / "driver.exe"
path.write_bytes(b"not a PE file")
with self.assertRaisesRegex(PeFormatError, "missing DOS header"):
imported_dlls(path)
if __name__ == "__main__":
unittest.main()