mirror of
https://github.com/t8y2/dbx.git
synced 2026-10-21 03:59:03 +00:00
@@ -84,6 +84,18 @@ test("builds a manually versioned module even without runtime file changes", ()
|
||||
assert.equal(result.versions.duckdb, "0.1.1");
|
||||
});
|
||||
|
||||
test("bumps DuckDB when its Cargo target configuration changes", () => {
|
||||
const result = evaluateAgentVersionBump({
|
||||
versions: { duckdb: "0.1.2" },
|
||||
changedFiles: ["agents/drivers/duckdb/.cargo/config.toml"],
|
||||
moduleExists: (path) => path === "agents/drivers/duckdb",
|
||||
readModuleFile: () => "",
|
||||
});
|
||||
|
||||
assert.equal(result.versions.duckdb, "0.1.3");
|
||||
assert.deepEqual(result.nativeModules, ["duckdb"]);
|
||||
});
|
||||
|
||||
test("builds only common-dependent Java modules for a shared runtime change", () => {
|
||||
const existing = new Set([
|
||||
"agents/drivers/access",
|
||||
|
||||
@@ -447,6 +447,10 @@ jobs:
|
||||
with:
|
||||
toolchain: nightly-2026-07-22
|
||||
components: rust-src
|
||||
- uses: actions/setup-python@v5
|
||||
if: runner.os == 'Windows'
|
||||
with:
|
||||
python-version: "3.13"
|
||||
- uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
|
||||
with:
|
||||
version: "v0.10.0"
|
||||
@@ -460,14 +464,14 @@ jobs:
|
||||
-e CARGO_INCREMENTAL=0 \
|
||||
-e CARGO_TARGET_DIR=/workspace/target/duckdb-driver \
|
||||
-v "${{ github.workspace }}:/workspace" \
|
||||
-w /workspace \
|
||||
-w /workspace/agents/drivers/duckdb \
|
||||
"${{ matrix.manylinux_image }}" \
|
||||
bash -lc '
|
||||
mkdir -p "$HOME"
|
||||
curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.97.1
|
||||
source "$HOME/.cargo/env"
|
||||
cargo build \
|
||||
--manifest-path agents/drivers/duckdb/Cargo.toml \
|
||||
--manifest-path Cargo.toml \
|
||||
--locked \
|
||||
--release \
|
||||
--bin dbx-duckdb-driver \
|
||||
@@ -478,18 +482,24 @@ jobs:
|
||||
if [ "${{ matrix.win7 }}" = "true" ]; then
|
||||
BUILD_STD=(-Z build-std=std,panic_abort)
|
||||
fi
|
||||
pushd agents/drivers/duckdb
|
||||
cargo build \
|
||||
--manifest-path agents/drivers/duckdb/Cargo.toml \
|
||||
--manifest-path Cargo.toml \
|
||||
--locked \
|
||||
--release \
|
||||
--bin dbx-duckdb-driver \
|
||||
--target "${{ matrix.target }}" \
|
||||
"${BUILD_STD[@]}"
|
||||
popd
|
||||
fi
|
||||
mkdir -p release-native
|
||||
cp \
|
||||
"target/duckdb-driver/${{ matrix.target }}/release/dbx-duckdb-driver${{ matrix.extension }}" \
|
||||
"release-native/dbx-agent-duckdb-${{ matrix.platform }}${{ matrix.extension }}"
|
||||
if [ "${{ runner.os }}" = "Windows" ]; then
|
||||
python agents/scripts/validate_windows_pe_dependencies.py \
|
||||
"release-native/dbx-agent-duckdb-${{ matrix.platform }}${{ matrix.extension }}"
|
||||
fi
|
||||
if [ -n "${{ matrix.manylinux_image }}" ]; then
|
||||
docker run --rm \
|
||||
--user "$(id -u):$(id -g)" \
|
||||
|
||||
@@ -148,6 +148,51 @@ jobs:
|
||||
}
|
||||
Get-FileHash -LiteralPath $installer.FullName -Algorithm SHA256
|
||||
|
||||
duckdb-windows-driver:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.duckdb_windows == 'true'
|
||||
runs-on: windows-2022
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
CARGO_INCREMENTAL: "0"
|
||||
CARGO_TARGET_DIR: ${{ github.workspace }}/target/duckdb-driver
|
||||
RUSTC_WRAPPER: sccache
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- name: Setup Rust for Windows 7
|
||||
uses: dtolnay/rust-toolchain@nightly
|
||||
with:
|
||||
toolchain: nightly-2026-07-22
|
||||
components: rust-src
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
|
||||
with:
|
||||
version: "v0.10.0"
|
||||
|
||||
- name: Build DuckDB Windows driver
|
||||
shell: bash
|
||||
run: |
|
||||
pushd agents/drivers/duckdb
|
||||
cargo build \
|
||||
--locked \
|
||||
--release \
|
||||
--bin dbx-duckdb-driver \
|
||||
--target x86_64-win7-windows-msvc \
|
||||
-Z build-std=std,panic_abort
|
||||
popd
|
||||
|
||||
- name: Validate DuckDB Windows driver
|
||||
shell: bash
|
||||
run: |
|
||||
DRIVER="target/duckdb-driver/x86_64-win7-windows-msvc/release/dbx-duckdb-driver.exe"
|
||||
python agents/scripts/validate_windows_pe_dependencies.py "$DRIVER"
|
||||
"$DRIVER" < /dev/null
|
||||
|
||||
rust-fmt-clippy:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
@@ -370,6 +415,7 @@ jobs:
|
||||
rust_full: ${{ steps.rust-mode.outputs.full }}
|
||||
jdbc: ${{ steps.filter.outputs.jdbc }}
|
||||
agents: ${{ steps.filter.outputs.agents }}
|
||||
duckdb_windows: ${{ steps.filter.outputs.duckdb_windows }}
|
||||
nix: ${{ steps.filter.outputs.nix }}
|
||||
windows_win7_bundle: ${{ steps.filter.outputs.windows_win7_bundle }}
|
||||
github_scripts: ${{ steps.filter.outputs.github_scripts }}
|
||||
@@ -421,6 +467,11 @@ jobs:
|
||||
- '.github/scripts/bump-agent-versions.test.mjs'
|
||||
- '.github/workflows/agents-release.yml'
|
||||
- '.github/workflows/ci.yml'
|
||||
duckdb_windows:
|
||||
- 'agents/drivers/duckdb/**'
|
||||
- 'agents/scripts/validate_windows_pe_dependencies.py'
|
||||
- '.github/workflows/agents-release.yml'
|
||||
- '.github/workflows/ci.yml'
|
||||
nix:
|
||||
# These advisory checks validate the pnpm and Cargo dependency closures.
|
||||
- 'package.json'
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
[target.'cfg(all(windows, target_env = "msvc"))']
|
||||
rustflags = ["-C", "target-feature=+crt-static"]
|
||||
@@ -7,7 +7,8 @@ existing newline-delimited JSON worker runtime, while keeping DuckDB and
|
||||
## Build
|
||||
|
||||
```bash
|
||||
cargo build --manifest-path agents/drivers/duckdb/Cargo.toml --release --bin dbx-duckdb-driver
|
||||
cd agents/drivers/duckdb
|
||||
cargo build --release --bin dbx-duckdb-driver
|
||||
```
|
||||
|
||||
Point DBX at the resulting executable with:
|
||||
@@ -35,6 +36,9 @@ DBX decompresses the package itself, so users do not need to install `zstd`,
|
||||
DuckDB, or a separate database driver. The existing aggregate offline `.zip`
|
||||
packages remain supported for backward compatibility.
|
||||
|
||||
Windows MSVC artifacts statically link the Visual C++ runtime so they also run
|
||||
on fresh Windows installations without a separate redistributable package.
|
||||
|
||||
## Current scope
|
||||
|
||||
The driver implements connect, execute, database/schema/table/column metadata,
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import struct
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class PeFormatError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _read_u16(data: bytes, offset: int) -> int:
|
||||
if offset < 0 or offset + 2 > len(data):
|
||||
raise PeFormatError("unexpected end of PE file")
|
||||
return struct.unpack_from("<H", data, offset)[0]
|
||||
|
||||
|
||||
def _read_u32(data: bytes, offset: int) -> int:
|
||||
if offset < 0 or offset + 4 > len(data):
|
||||
raise PeFormatError("unexpected end of PE file")
|
||||
return struct.unpack_from("<I", data, offset)[0]
|
||||
|
||||
|
||||
def _read_c_string(data: bytes, offset: int) -> str:
|
||||
if offset < 0 or offset >= len(data):
|
||||
raise PeFormatError("PE string offset is outside the file")
|
||||
end = data.find(b"\0", offset)
|
||||
if end < 0:
|
||||
raise PeFormatError("unterminated PE string")
|
||||
try:
|
||||
return data[offset:end].decode("ascii")
|
||||
except UnicodeDecodeError as error:
|
||||
raise PeFormatError("PE import name is not ASCII") from error
|
||||
|
||||
|
||||
def imported_dlls(path: Path) -> list[str]:
|
||||
data = path.read_bytes()
|
||||
if len(data) < 64 or data[:2] != b"MZ":
|
||||
raise PeFormatError("missing DOS header")
|
||||
|
||||
pe_offset = _read_u32(data, 0x3C)
|
||||
if data[pe_offset : pe_offset + 4] != b"PE\0\0":
|
||||
raise PeFormatError("missing PE signature")
|
||||
|
||||
section_count = _read_u16(data, pe_offset + 6)
|
||||
optional_header_size = _read_u16(data, pe_offset + 20)
|
||||
optional_header_offset = pe_offset + 24
|
||||
optional_magic = _read_u16(data, optional_header_offset)
|
||||
if optional_magic == 0x20B:
|
||||
data_directories_offset = optional_header_offset + 112
|
||||
elif optional_magic == 0x10B:
|
||||
data_directories_offset = optional_header_offset + 96
|
||||
else:
|
||||
raise PeFormatError(f"unsupported PE optional header magic: 0x{optional_magic:04x}")
|
||||
|
||||
import_directory_rva = _read_u32(data, data_directories_offset + 8)
|
||||
import_directory_size = _read_u32(data, data_directories_offset + 12)
|
||||
if import_directory_rva == 0 or import_directory_size == 0:
|
||||
return []
|
||||
|
||||
size_of_headers = _read_u32(data, optional_header_offset + 60)
|
||||
section_table_offset = optional_header_offset + optional_header_size
|
||||
sections = []
|
||||
for index in range(section_count):
|
||||
section_offset = section_table_offset + index * 40
|
||||
sections.append(
|
||||
(
|
||||
_read_u32(data, section_offset + 12),
|
||||
_read_u32(data, section_offset + 8),
|
||||
_read_u32(data, section_offset + 20),
|
||||
_read_u32(data, section_offset + 16),
|
||||
)
|
||||
)
|
||||
|
||||
def rva_to_offset(rva: int) -> int:
|
||||
if rva < size_of_headers:
|
||||
return rva
|
||||
for virtual_address, virtual_size, raw_offset, raw_size in sections:
|
||||
span = max(virtual_size, raw_size)
|
||||
if virtual_address <= rva < virtual_address + span:
|
||||
delta = rva - virtual_address
|
||||
if delta >= raw_size:
|
||||
break
|
||||
return raw_offset + delta
|
||||
raise PeFormatError(f"PE RVA 0x{rva:x} is not backed by file data")
|
||||
|
||||
descriptor_offset = rva_to_offset(import_directory_rva)
|
||||
imports = []
|
||||
for _ in range(4096):
|
||||
if descriptor_offset + 20 > len(data):
|
||||
raise PeFormatError("truncated PE import descriptor")
|
||||
descriptor = data[descriptor_offset : descriptor_offset + 20]
|
||||
if descriptor == b"\0" * 20:
|
||||
return sorted(set(imports), key=str.casefold)
|
||||
name_rva = _read_u32(data, descriptor_offset + 12)
|
||||
if name_rva == 0:
|
||||
raise PeFormatError("PE import descriptor has no DLL name")
|
||||
imports.append(_read_c_string(data, rva_to_offset(name_rva)))
|
||||
descriptor_offset += 20
|
||||
|
||||
raise PeFormatError("PE import descriptor table is not terminated")
|
||||
|
||||
|
||||
def forbidden_msvc_runtime_dlls(imports: list[str]) -> list[str]:
|
||||
return sorted(
|
||||
{name for name in imports if name.casefold().startswith(("msvcp", "vcruntime"))},
|
||||
key=str.casefold,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Reject Windows PE files that require the Visual C++ runtime")
|
||||
parser.add_argument("binary", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
imports = imported_dlls(args.binary)
|
||||
except (OSError, PeFormatError) as error:
|
||||
parser.error(str(error))
|
||||
|
||||
forbidden = forbidden_msvc_runtime_dlls(imports)
|
||||
if forbidden:
|
||||
parser.error(f"dynamic Visual C++ runtime dependencies found: {', '.join(forbidden)}")
|
||||
|
||||
print(f"Validated {args.binary}: {len(imports)} imported DLLs, no dynamic Visual C++ runtime")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,76 @@
|
||||
import struct
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from validate_windows_pe_dependencies import PeFormatError, forbidden_msvc_runtime_dlls, imported_dlls
|
||||
|
||||
|
||||
def build_test_pe(imports: list[str]) -> bytes:
|
||||
data = bytearray(4096)
|
||||
pe_offset = 0x80
|
||||
optional_header_offset = pe_offset + 24
|
||||
optional_header_size = 0xF0
|
||||
section_table_offset = optional_header_offset + optional_header_size
|
||||
section_rva = 0x1000
|
||||
section_offset = 0x200
|
||||
import_directory_rva = section_rva
|
||||
|
||||
data[:2] = b"MZ"
|
||||
struct.pack_into("<I", data, 0x3C, pe_offset)
|
||||
data[pe_offset : pe_offset + 4] = b"PE\0\0"
|
||||
struct.pack_into("<H", data, pe_offset + 4, 0x8664)
|
||||
struct.pack_into("<H", data, pe_offset + 6, 1)
|
||||
struct.pack_into("<H", data, pe_offset + 20, optional_header_size)
|
||||
struct.pack_into("<H", data, optional_header_offset, 0x20B)
|
||||
struct.pack_into("<I", data, optional_header_offset + 60, section_offset)
|
||||
struct.pack_into("<I", data, optional_header_offset + 112 + 8, import_directory_rva)
|
||||
struct.pack_into("<I", data, optional_header_offset + 112 + 12, (len(imports) + 1) * 20)
|
||||
|
||||
data[section_table_offset : section_table_offset + 8] = b".rdata\0\0"
|
||||
struct.pack_into("<I", data, section_table_offset + 8, 0x800)
|
||||
struct.pack_into("<I", data, section_table_offset + 12, section_rva)
|
||||
struct.pack_into("<I", data, section_table_offset + 16, 0x800)
|
||||
struct.pack_into("<I", data, section_table_offset + 20, section_offset)
|
||||
|
||||
name_offset = section_offset + 0x200
|
||||
for index, name in enumerate(imports):
|
||||
descriptor_offset = section_offset + index * 20
|
||||
name_rva = section_rva + (name_offset - section_offset)
|
||||
struct.pack_into("<I", data, descriptor_offset + 12, name_rva)
|
||||
encoded = name.encode("ascii") + b"\0"
|
||||
data[name_offset : name_offset + len(encoded)] = encoded
|
||||
name_offset += len(encoded)
|
||||
|
||||
return bytes(data)
|
||||
|
||||
|
||||
class WindowsPeDependenciesTest(unittest.TestCase):
|
||||
def test_reads_imported_dll_names(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
path = Path(temp_dir) / "driver.exe"
|
||||
path.write_bytes(build_test_pe(["KERNEL32.dll", "bcrypt.dll"]))
|
||||
|
||||
self.assertEqual(imported_dlls(path), ["bcrypt.dll", "KERNEL32.dll"])
|
||||
|
||||
def test_rejects_dynamic_visual_cpp_runtime(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
path = Path(temp_dir) / "driver.exe"
|
||||
path.write_bytes(build_test_pe(["KERNEL32.dll", "MSVCP140.dll", "VCRUNTIME140_1.dll"]))
|
||||
|
||||
self.assertEqual(
|
||||
forbidden_msvc_runtime_dlls(imported_dlls(path)),
|
||||
["MSVCP140.dll", "VCRUNTIME140_1.dll"],
|
||||
)
|
||||
|
||||
def test_rejects_non_pe_files(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
path = Path(temp_dir) / "driver.exe"
|
||||
path.write_bytes(b"not a PE file")
|
||||
|
||||
with self.assertRaisesRegex(PeFormatError, "missing DOS header"):
|
||||
imported_dlls(path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user