From cd5c222f95424dcb6fddfe1bf5e9348744223a7b Mon Sep 17 00:00:00 2001 From: t8y2 <1156263951@qq.com> Date: Tue, 4 Aug 2026 15:08:04 +0800 Subject: [PATCH] fix(duckdb): statically link Windows worker runtime Closes #5313 --- .github/scripts/bump-agent-versions.test.mjs | 12 ++ .github/workflows/agents-release.yml | 16 ++- .github/workflows/ci.yml | 51 +++++++ agents/drivers/duckdb/.cargo/config.toml | 2 + agents/drivers/duckdb/README.md | 6 +- .../validate_windows_pe_dependencies.py | 130 ++++++++++++++++++ .../validate_windows_pe_dependencies_test.py | 76 ++++++++++ 7 files changed, 289 insertions(+), 4 deletions(-) create mode 100644 agents/drivers/duckdb/.cargo/config.toml create mode 100644 agents/scripts/validate_windows_pe_dependencies.py create mode 100644 agents/scripts/validate_windows_pe_dependencies_test.py diff --git a/.github/scripts/bump-agent-versions.test.mjs b/.github/scripts/bump-agent-versions.test.mjs index 5d58964d50..98c5bc74a6 100644 --- a/.github/scripts/bump-agent-versions.test.mjs +++ b/.github/scripts/bump-agent-versions.test.mjs @@ -84,6 +84,18 @@ test("builds a manually versioned module even without runtime file changes", () assert.equal(result.versions.duckdb, "0.1.1"); }); +test("bumps DuckDB when its Cargo target configuration changes", () => { + const result = evaluateAgentVersionBump({ + versions: { duckdb: "0.1.2" }, + changedFiles: ["agents/drivers/duckdb/.cargo/config.toml"], + moduleExists: (path) => path === "agents/drivers/duckdb", + readModuleFile: () => "", + }); + + assert.equal(result.versions.duckdb, "0.1.3"); + assert.deepEqual(result.nativeModules, ["duckdb"]); +}); + test("builds only common-dependent Java modules for a shared runtime change", () => { const existing = new Set([ "agents/drivers/access", diff --git a/.github/workflows/agents-release.yml b/.github/workflows/agents-release.yml index 6d2188ff08..8c55f41766 100644 --- a/.github/workflows/agents-release.yml +++ b/.github/workflows/agents-release.yml @@ -447,6 +447,10 @@ jobs: with: toolchain: nightly-2026-07-22 components: rust-src + - uses: actions/setup-python@v5 + if: runner.os == 'Windows' + with: + python-version: "3.13" - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 with: version: "v0.10.0" @@ -460,14 +464,14 @@ jobs: -e CARGO_INCREMENTAL=0 \ -e CARGO_TARGET_DIR=/workspace/target/duckdb-driver \ -v "${{ github.workspace }}:/workspace" \ - -w /workspace \ + -w /workspace/agents/drivers/duckdb \ "${{ matrix.manylinux_image }}" \ bash -lc ' mkdir -p "$HOME" curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.97.1 source "$HOME/.cargo/env" cargo build \ - --manifest-path agents/drivers/duckdb/Cargo.toml \ + --manifest-path Cargo.toml \ --locked \ --release \ --bin dbx-duckdb-driver \ @@ -478,18 +482,24 @@ jobs: if [ "${{ matrix.win7 }}" = "true" ]; then BUILD_STD=(-Z build-std=std,panic_abort) fi + pushd agents/drivers/duckdb cargo build \ - --manifest-path agents/drivers/duckdb/Cargo.toml \ + --manifest-path Cargo.toml \ --locked \ --release \ --bin dbx-duckdb-driver \ --target "${{ matrix.target }}" \ "${BUILD_STD[@]}" + popd fi mkdir -p release-native cp \ "target/duckdb-driver/${{ matrix.target }}/release/dbx-duckdb-driver${{ matrix.extension }}" \ "release-native/dbx-agent-duckdb-${{ matrix.platform }}${{ matrix.extension }}" + if [ "${{ runner.os }}" = "Windows" ]; then + python agents/scripts/validate_windows_pe_dependencies.py \ + "release-native/dbx-agent-duckdb-${{ matrix.platform }}${{ matrix.extension }}" + fi if [ -n "${{ matrix.manylinux_image }}" ]; then docker run --rm \ --user "$(id -u):$(id -g)" \ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8c1d70361..99c0064f29 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,6 +148,51 @@ jobs: } Get-FileHash -LiteralPath $installer.FullName -Algorithm SHA256 + duckdb-windows-driver: + needs: changes + if: needs.changes.outputs.duckdb_windows == 'true' + runs-on: windows-2022 + timeout-minutes: 60 + env: + CARGO_INCREMENTAL: "0" + CARGO_TARGET_DIR: ${{ github.workspace }}/target/duckdb-driver + RUSTC_WRAPPER: sccache + steps: + - uses: actions/checkout@v5 + + - name: Setup Rust for Windows 7 + uses: dtolnay/rust-toolchain@nightly + with: + toolchain: nightly-2026-07-22 + components: rust-src + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + with: + version: "v0.10.0" + + - name: Build DuckDB Windows driver + shell: bash + run: | + pushd agents/drivers/duckdb + cargo build \ + --locked \ + --release \ + --bin dbx-duckdb-driver \ + --target x86_64-win7-windows-msvc \ + -Z build-std=std,panic_abort + popd + + - name: Validate DuckDB Windows driver + shell: bash + run: | + DRIVER="target/duckdb-driver/x86_64-win7-windows-msvc/release/dbx-duckdb-driver.exe" + python agents/scripts/validate_windows_pe_dependencies.py "$DRIVER" + "$DRIVER" < /dev/null + rust-fmt-clippy: needs: changes if: needs.changes.outputs.rust == 'true' @@ -370,6 +415,7 @@ jobs: rust_full: ${{ steps.rust-mode.outputs.full }} jdbc: ${{ steps.filter.outputs.jdbc }} agents: ${{ steps.filter.outputs.agents }} + duckdb_windows: ${{ steps.filter.outputs.duckdb_windows }} nix: ${{ steps.filter.outputs.nix }} windows_win7_bundle: ${{ steps.filter.outputs.windows_win7_bundle }} github_scripts: ${{ steps.filter.outputs.github_scripts }} @@ -421,6 +467,11 @@ jobs: - '.github/scripts/bump-agent-versions.test.mjs' - '.github/workflows/agents-release.yml' - '.github/workflows/ci.yml' + duckdb_windows: + - 'agents/drivers/duckdb/**' + - 'agents/scripts/validate_windows_pe_dependencies.py' + - '.github/workflows/agents-release.yml' + - '.github/workflows/ci.yml' nix: # These advisory checks validate the pnpm and Cargo dependency closures. - 'package.json' diff --git a/agents/drivers/duckdb/.cargo/config.toml b/agents/drivers/duckdb/.cargo/config.toml new file mode 100644 index 0000000000..35c67ad3d2 --- /dev/null +++ b/agents/drivers/duckdb/.cargo/config.toml @@ -0,0 +1,2 @@ +[target.'cfg(all(windows, target_env = "msvc"))'] +rustflags = ["-C", "target-feature=+crt-static"] diff --git a/agents/drivers/duckdb/README.md b/agents/drivers/duckdb/README.md index c24550cbeb..24be78fa3b 100644 --- a/agents/drivers/duckdb/README.md +++ b/agents/drivers/duckdb/README.md @@ -7,7 +7,8 @@ existing newline-delimited JSON worker runtime, while keeping DuckDB and ## Build ```bash -cargo build --manifest-path agents/drivers/duckdb/Cargo.toml --release --bin dbx-duckdb-driver +cd agents/drivers/duckdb +cargo build --release --bin dbx-duckdb-driver ``` Point DBX at the resulting executable with: @@ -35,6 +36,9 @@ DBX decompresses the package itself, so users do not need to install `zstd`, DuckDB, or a separate database driver. The existing aggregate offline `.zip` packages remain supported for backward compatibility. +Windows MSVC artifacts statically link the Visual C++ runtime so they also run +on fresh Windows installations without a separate redistributable package. + ## Current scope The driver implements connect, execute, database/schema/table/column metadata, diff --git a/agents/scripts/validate_windows_pe_dependencies.py b/agents/scripts/validate_windows_pe_dependencies.py new file mode 100644 index 0000000000..2d10d4e772 --- /dev/null +++ b/agents/scripts/validate_windows_pe_dependencies.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 + +import argparse +import struct +from pathlib import Path + + +class PeFormatError(ValueError): + pass + + +def _read_u16(data: bytes, offset: int) -> int: + if offset < 0 or offset + 2 > len(data): + raise PeFormatError("unexpected end of PE file") + return struct.unpack_from(" int: + if offset < 0 or offset + 4 > len(data): + raise PeFormatError("unexpected end of PE file") + return struct.unpack_from(" str: + if offset < 0 or offset >= len(data): + raise PeFormatError("PE string offset is outside the file") + end = data.find(b"\0", offset) + if end < 0: + raise PeFormatError("unterminated PE string") + try: + return data[offset:end].decode("ascii") + except UnicodeDecodeError as error: + raise PeFormatError("PE import name is not ASCII") from error + + +def imported_dlls(path: Path) -> list[str]: + data = path.read_bytes() + if len(data) < 64 or data[:2] != b"MZ": + raise PeFormatError("missing DOS header") + + pe_offset = _read_u32(data, 0x3C) + if data[pe_offset : pe_offset + 4] != b"PE\0\0": + raise PeFormatError("missing PE signature") + + section_count = _read_u16(data, pe_offset + 6) + optional_header_size = _read_u16(data, pe_offset + 20) + optional_header_offset = pe_offset + 24 + optional_magic = _read_u16(data, optional_header_offset) + if optional_magic == 0x20B: + data_directories_offset = optional_header_offset + 112 + elif optional_magic == 0x10B: + data_directories_offset = optional_header_offset + 96 + else: + raise PeFormatError(f"unsupported PE optional header magic: 0x{optional_magic:04x}") + + import_directory_rva = _read_u32(data, data_directories_offset + 8) + import_directory_size = _read_u32(data, data_directories_offset + 12) + if import_directory_rva == 0 or import_directory_size == 0: + return [] + + size_of_headers = _read_u32(data, optional_header_offset + 60) + section_table_offset = optional_header_offset + optional_header_size + sections = [] + for index in range(section_count): + section_offset = section_table_offset + index * 40 + sections.append( + ( + _read_u32(data, section_offset + 12), + _read_u32(data, section_offset + 8), + _read_u32(data, section_offset + 20), + _read_u32(data, section_offset + 16), + ) + ) + + def rva_to_offset(rva: int) -> int: + if rva < size_of_headers: + return rva + for virtual_address, virtual_size, raw_offset, raw_size in sections: + span = max(virtual_size, raw_size) + if virtual_address <= rva < virtual_address + span: + delta = rva - virtual_address + if delta >= raw_size: + break + return raw_offset + delta + raise PeFormatError(f"PE RVA 0x{rva:x} is not backed by file data") + + descriptor_offset = rva_to_offset(import_directory_rva) + imports = [] + for _ in range(4096): + if descriptor_offset + 20 > len(data): + raise PeFormatError("truncated PE import descriptor") + descriptor = data[descriptor_offset : descriptor_offset + 20] + if descriptor == b"\0" * 20: + return sorted(set(imports), key=str.casefold) + name_rva = _read_u32(data, descriptor_offset + 12) + if name_rva == 0: + raise PeFormatError("PE import descriptor has no DLL name") + imports.append(_read_c_string(data, rva_to_offset(name_rva))) + descriptor_offset += 20 + + raise PeFormatError("PE import descriptor table is not terminated") + + +def forbidden_msvc_runtime_dlls(imports: list[str]) -> list[str]: + return sorted( + {name for name in imports if name.casefold().startswith(("msvcp", "vcruntime"))}, + key=str.casefold, + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Reject Windows PE files that require the Visual C++ runtime") + parser.add_argument("binary", type=Path) + args = parser.parse_args() + + try: + imports = imported_dlls(args.binary) + except (OSError, PeFormatError) as error: + parser.error(str(error)) + + forbidden = forbidden_msvc_runtime_dlls(imports) + if forbidden: + parser.error(f"dynamic Visual C++ runtime dependencies found: {', '.join(forbidden)}") + + print(f"Validated {args.binary}: {len(imports)} imported DLLs, no dynamic Visual C++ runtime") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/agents/scripts/validate_windows_pe_dependencies_test.py b/agents/scripts/validate_windows_pe_dependencies_test.py new file mode 100644 index 0000000000..237a8acc91 --- /dev/null +++ b/agents/scripts/validate_windows_pe_dependencies_test.py @@ -0,0 +1,76 @@ +import struct +import tempfile +import unittest +from pathlib import Path + +from validate_windows_pe_dependencies import PeFormatError, forbidden_msvc_runtime_dlls, imported_dlls + + +def build_test_pe(imports: list[str]) -> bytes: + data = bytearray(4096) + pe_offset = 0x80 + optional_header_offset = pe_offset + 24 + optional_header_size = 0xF0 + section_table_offset = optional_header_offset + optional_header_size + section_rva = 0x1000 + section_offset = 0x200 + import_directory_rva = section_rva + + data[:2] = b"MZ" + struct.pack_into(" None: + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "driver.exe" + path.write_bytes(build_test_pe(["KERNEL32.dll", "bcrypt.dll"])) + + self.assertEqual(imported_dlls(path), ["bcrypt.dll", "KERNEL32.dll"]) + + def test_rejects_dynamic_visual_cpp_runtime(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "driver.exe" + path.write_bytes(build_test_pe(["KERNEL32.dll", "MSVCP140.dll", "VCRUNTIME140_1.dll"])) + + self.assertEqual( + forbidden_msvc_runtime_dlls(imported_dlls(path)), + ["MSVCP140.dll", "VCRUNTIME140_1.dll"], + ) + + def test_rejects_non_pe_files(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "driver.exe" + path.write_bytes(b"not a PE file") + + with self.assertRaisesRegex(PeFormatError, "missing DOS header"): + imported_dlls(path) + + +if __name__ == "__main__": + unittest.main()