diff --git a/.github/scripts/ci-agent-go.sh b/.github/scripts/ci-agent-go.sh new file mode 100644 index 0000000000..e72f1bc696 --- /dev/null +++ b/.github/scripts/ci-agent-go.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +driver="${1:?driver is required}" +binary="${2:?binary name is required}" +race="${3:?race mode is required}" +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$root/agents/drivers/$driver" +if [ "$driver" = xugu ]; then export GONOSUMDB=gitee.com/XuguDB/go-xugu-driver; fi + +if [ "$race" = true ]; then go test -race ./...; else go test ./...; fi +if [ "$driver" = zookeeper ]; then + for target in darwin/arm64 darwin/amd64 linux/arm64 linux/amd64 windows/arm64 windows/amd64; do + IFS=/ read -r goos goarch <<< "$target" + output="/tmp/dbx-agent-zookeeper-${goos}-${goarch}" + [ "$goos" = windows ] && output="${output}.exe" + CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags="-s -w" -o "$output" . + done +else + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o "/tmp/dbx-agent-${binary}-linux-x64" . +fi diff --git a/.github/scripts/ci-agent-integration.sh b/.github/scripts/ci-agent-integration.sh new file mode 100644 index 0000000000..7ff5ac2367 --- /dev/null +++ b/.github/scripts/ci-agent-integration.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +set -euo pipefail + +scenario="${1:?scenario is required}" +version="${2:?version is required}" +image="${3:-}" +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +case "$scenario" in + zookeeper) + cd "$root/agents/drivers/zookeeper" + name="dbx-zookeeper-${version//./-}" + docker rm -fv "$name" >/dev/null 2>&1 || true + docker run -d --name "$name" -p 2181:2181 "zookeeper:$version" + cleanup() { + docker rm -fv "$name" >/dev/null 2>&1 || true + } + trap cleanup EXIT + ready=false + for _ in $(seq 1 90); do + if timeout 1 bash -c '/dev/null 2>&1; then + ready=true + break + fi + sleep 2 + done + if [ "$ready" != true ]; then + docker logs "$name" + exit 1 + fi + DBX_ZOOKEEPER_TEST_CONNECT_STRING=127.0.0.1:2181 \ + go test -run '^TestZooKeeperIntegration$' -count=1 ./... + if [ "$version" = "3.7.0" ]; then + DBX_ZOOKEEPER_TEST_CONNECT_STRING=127.0.0.1:2181 \ + go test -run '^TestZooKeeperLargeChildrenIntegration$' -count=1 ./... + fi + ;; + zookeeper-sasl) + cd "$root/agents/drivers/zookeeper" + name="dbx-zookeeper-sasl-3-7-0" + jaas_file=$(mktemp) + cat > "$jaas_file" <<'EOF' + Server { + org.apache.zookeeper.server.auth.DigestLoginModule required + user_dbx="dbx-secret"; + }; +EOF + chmod 0644 "$jaas_file" + docker rm -fv "$name" >/dev/null 2>&1 || true + docker run -d --name "$name" -p 2181:2181 \ + -v "$jaas_file:/conf/server-jaas.conf:ro" \ + -e ZOO_CFG_EXTRA="authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider" \ + -e JVMFLAGS="-Djava.security.auth.login.config=/conf/server-jaas.conf -Dzookeeper.sasl.serverconfig=Server -Dzookeeper.sessionRequireClientSASLAuth=true" \ + zookeeper:3.7.0 + cleanup() { + docker rm -fv "$name" >/dev/null 2>&1 || true + } + trap cleanup EXIT + ready=false + for _ in $(seq 1 90); do + if timeout 1 bash -c '/dev/null 2>&1; then + ready=true + break + fi + sleep 2 + done + if [ "$ready" != true ]; then + docker logs "$name" + exit 1 + fi + DBX_ZOOKEEPER_TEST_CONNECT_STRING=127.0.0.1:2181 \ + DBX_ZOOKEEPER_TEST_AUTH_SCHEME=sasl_digest \ + DBX_ZOOKEEPER_TEST_USERNAME=dbx \ + DBX_ZOOKEEPER_TEST_PASSWORD=dbx-secret \ + go test -run '^TestZooKeeperIntegration$' -count=1 ./... + ;; + rocketmq) + cd "$root/agents/drivers/rocketmq" + ./scripts/run-integration.sh "$version" + ;; + tdengine) + cd "$root/agents" + name="dbx-tdengine-${version//./-}" + docker rm -fv "$name" >/dev/null 2>&1 || true + docker run -d --name "$name" \ + --memory 4g \ + --ulimit nofile=65535:65535 \ + -p 6030:6030 \ + -p 6041:6041 \ + "$image" + cleanup() { + docker rm -fv "$name" >/dev/null 2>&1 || true + } + trap cleanup EXIT + ready=false + dnodes="" + for _ in $(seq 1 90); do + dnodes="$(docker exec "$name" taos -s 'SHOW DNODES' 2>/dev/null || true)" + if grep -Eq '\|[[:space:]]*ready[[:space:]]*\|' <<<"$dnodes"; then + ready=true + break + fi + sleep 2 + done + if [ "$ready" != "true" ]; then + printf '%s\n' "$dnodes" + docker logs "$name" + exit 1 + fi + TDENGINE_INTEGRATION=1 \ + TDENGINE_TEST_HOST=127.0.0.1 \ + TDENGINE_TEST_PORT=6041 \ + cargo test --manifest-path drivers/tdengine/Cargo.toml --locked --test live -- --nocapture + ;; + cassandra) + cd "$root/agents/drivers/cassandra-go" + name="dbx-cassandra-${version//./-}" + docker rm -fv "$name" >/dev/null 2>&1 || true + docker run -d --name "$name" \ + -e CASSANDRA_CLUSTER_NAME="DBX Cassandra CI $version" \ + -e CASSANDRA_DC=dc1 \ + -e CASSANDRA_RACK=rack1 \ + -e CASSANDRA_ENDPOINT_SNITCH=GossipingPropertyFileSnitch \ + -e CASSANDRA_NUM_TOKENS=16 \ + -e MAX_HEAP_SIZE=512M \ + -e HEAP_NEWSIZE=100M \ + -p 9042:9042 \ + "cassandra:$version" + cleanup() { + docker rm -fv "$name" >/dev/null 2>&1 || true + } + trap cleanup EXIT + ready=false + for _ in $(seq 1 100); do + if docker exec "$name" cqlsh -e 'SELECT release_version FROM system.local' >/dev/null 2>&1; then + ready=true + break + fi + sleep 3 + done + if [ "$ready" != "true" ]; then + docker logs "$name" + exit 1 + fi + CASSANDRA_TEST_HOST=127.0.0.1 \ + CASSANDRA_TEST_PORT=9042 \ + go test -run '^TestCassandraIntegration$' -count=1 ./... + ;; + rabbitmq) + cd "$root/agents/drivers/rabbitmq" + name="dbx-rabbitmq-${version//./-}" + cookie="dbx-ci-${GITHUB_RUN_ID:-local}-${version//./-}" + # RabbitMQ is disposable in CI. Keep its data on a fresh tmpfs + # owned by the image's rabbitmq user so .erlang.cookie is readable. + docker rm -fv "$name" >/dev/null 2>&1 || true + docker run -d --name "$name" \ + --user 999:999 \ + --tmpfs /var/lib/rabbitmq:rw,exec,uid=999,gid=999,mode=700 \ + -e RABBITMQ_DEFAULT_USER=dbx \ + -e RABBITMQ_DEFAULT_PASS=dbx-password \ + -e RABBITMQ_ERLANG_COOKIE="$cookie" \ + -p 5672:5672 -p 15672:15672 \ + "rabbitmq:${version}-management" + cleanup() { + docker rm -fv "$name" >/dev/null 2>&1 || true + } + trap cleanup EXIT + ready=false + for _ in $(seq 1 60); do + if docker exec "$name" rabbitmq-diagnostics -q check_running >/dev/null 2>&1 \ + && curl --fail --silent --noproxy '*' --user dbx:dbx-password \ + http://127.0.0.1:15672/api/overview >/dev/null; then + ready=true + break + fi + sleep 2 + done + if [ "$ready" != "true" ]; then + docker inspect "$name" --format 'image={{.Config.Image}} user={{.Config.User}} status={{.State.Status}} exit={{.State.ExitCode}}' || true + docker logs "$name" + exit 1 + fi + RABBITMQ_INTEGRATION=1 \ + RABBITMQ_USERNAME=dbx \ + RABBITMQ_PASSWORD=dbx-password \ + go test -run '^TestRabbitMQIntegration$' -count=1 ./... + ;; + *) + echo "Unknown integration scenario: $scenario" >&2 + exit 2 + ;; +esac diff --git a/.github/scripts/ci-config.mjs b/.github/scripts/ci-config.mjs new file mode 100644 index 0000000000..0283dbb63b --- /dev/null +++ b/.github/scripts/ci-config.mjs @@ -0,0 +1,32 @@ +export const rustGroups = { + foundation: ["dbx-types", "dbx-platform", "dbx-sql", "dbx-formats", "dbx-ai-provider", "dbx-plugin-runtime"], + drivers: ["dbx-drivers", "dbx-sqlite-worker"], + application: ["dbx", "dbx-core", "dbx-web", "dbx-cli", "dbx-mcp"], +}; + +export const goAgents = [ + { driver: "oracle-go", binary: "oracle", race: false }, + { driver: "xugu", binary: "xugu", race: false }, + { driver: "rabbitmq", binary: "rabbitmq", race: false }, + { driver: "rocketmq", binary: "rocketmq", race: true }, + { driver: "zookeeper", binary: "zookeeper", race: true }, + { driver: "cassandra-go", binary: "cassandra", race: false }, + { driver: "hive-go", binary: "hive", race: false }, + { driver: "vastbase-go", binary: "vastbase", race: false }, + { driver: "neo4j-go", binary: "neo4j", race: false }, + { driver: "iotdb", binary: "iotdb", race: false }, +]; + +export const rustAgents = ["duckdb", "tdengine"]; + +export const integrationCases = [ + ...["3.4.14", "3.5.5", "3.7.0", "3.9.5"].map((version) => ({ driver: "zookeeper", scenario: "zookeeper", version })), + { driver: "zookeeper", scenario: "zookeeper-sasl", version: "3.7.0" }, + ...["4.9.8", "5.3.1"].map((version) => ({ driver: "rocketmq", scenario: "rocketmq", version })), + ...["2.4.0.14", "2.6.0.34", "3.0.7.1", "3.3.6.13", "3.4.2.2"].map((version) => ({ + driver: "tdengine", scenario: "tdengine", version, + image: `tdengine/${version === "3.4.2.2" ? "tsdb" : "tdengine"}:${version}`, + })), + ...["3.11.19", "5.0.6"].map((version) => ({ driver: "cassandra-go", scenario: "cassandra", version })), + ...["3.13", "4.3"].map((version) => ({ driver: "rabbitmq", scenario: "rabbitmq", version })), +]; diff --git a/.github/scripts/ci-execution.test.mjs b/.github/scripts/ci-execution.test.mjs new file mode 100644 index 0000000000..b8d37a853e --- /dev/null +++ b/.github/scripts/ci-execution.test.mjs @@ -0,0 +1,121 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; +import { goAgents, integrationCases } from "./ci-config.mjs"; +import { lockfileManifests } from "./ci-lockfiles.mjs"; + +const root = fileURLToPath(new URL("../../", import.meta.url)); +const environmentKeys = ["CGO_ENABLED", "GOOS", "GOARCH", "GONOSUMDB", "DBX_ZOOKEEPER_TEST_CONNECT_STRING", + "DBX_ZOOKEEPER_TEST_AUTH_SCHEME", "DBX_ZOOKEEPER_TEST_USERNAME", "DBX_ZOOKEEPER_TEST_PASSWORD", "TDENGINE_INTEGRATION", + "TDENGINE_TEST_HOST", "TDENGINE_TEST_PORT", "CASSANDRA_TEST_HOST", "CASSANDRA_TEST_PORT", "RABBITMQ_INTEGRATION", + "RABBITMQ_USERNAME", "RABBITMQ_PASSWORD"]; + +function withToolStubs(run, extraEnv = {}) { + const directory = mkdtempSync(path.join(tmpdir(), "dbx-ci-execution-")); + const binaries = path.join(directory, "bin"); + mkdirSync(binaries); + const log = path.join(directory, "commands.jsonl"); + const stub = path.join(binaries, "stub.cjs"); + writeFileSync(stub, `#!/usr/bin/env node +const fs = require('node:fs'); +const path = require('node:path'); +const tool = path.basename(process.argv[1]); +const args = process.argv.slice(2); +const env = Object.fromEntries(${JSON.stringify(environmentKeys)}.filter(name => process.env[name] !== undefined).map(name => [name, process.env[name]])); +fs.appendFileSync(process.env.CI_STUB_LOG, JSON.stringify({tool,args,cwd:process.cwd(),env})+'\\n'); +if (process.env.CI_STUB_FAIL === tool) process.exit(23); +if (tool === 'docker' && args.includes('taos')) console.log('| ready |'); +`, { mode: 0o755 }); + for (const name of ["docker", "go", "cargo", "timeout", "curl", "sleep", "bash"]) symlinkSync(stub, path.join(binaries, name)); + try { + const env = { ...process.env, ...extraEnv, TMPDIR: directory, CI_STUB_LOG: log, PATH: `${binaries}${path.delimiter}${process.env.PATH}` }; + return run(env, () => readFileSync(log, "utf8").trim().split("\n").filter(Boolean).map((line) => JSON.parse(line))); + } finally { rmSync(directory, { recursive: true, force: true }); } +} + +for (const agent of goAgents) { + test(`Go job preserves ${agent.driver} tests and target builds without executing compilers`, () => withToolStubs((env, commands) => { + const result = spawnSync("/bin/bash", [".github/scripts/ci-agent-go.sh", agent.driver, agent.binary, String(agent.race)], { cwd: root, env, encoding: "utf8" }); + assert.equal(result.status, 0, result.stderr); + const invocations = commands(); + assert.deepEqual(invocations[0].args, agent.race ? ["test", "-race", "./..."] : ["test", "./..."]); + assert.ok(invocations.every((entry) => entry.cwd === path.join(root, "agents/drivers", agent.driver))); + const builds = invocations.slice(1); + assert.equal(builds.length, agent.driver === "zookeeper" ? 6 : 1); + for (const build of builds) { + assert.deepEqual(build.args.slice(0, 3), ["build", "-trimpath", "-ldflags=-s -w"]); + assert.equal(build.env.CGO_ENABLED, "0"); + } + if (agent.driver === "zookeeper") assert.deepEqual(builds.map((entry) => `${entry.env.GOOS}/${entry.env.GOARCH}`), + ["darwin/arm64", "darwin/amd64", "linux/arm64", "linux/amd64", "windows/arm64", "windows/amd64"]); + else assert.ok(builds[0].args.includes(`/tmp/dbx-agent-${agent.binary}-linux-x64`)); + if (agent.driver === "xugu") assert.ok(invocations.every((entry) => entry.env.GONOSUMDB === "gitee.com/XuguDB/go-xugu-driver")); + })); +} + +for (const entry of integrationCases) { + test(`live matrix retains ${entry.scenario}/${entry.version} commands without starting any container`, () => withToolStubs((env, commands) => { + const result = spawnSync("/bin/bash", [".github/scripts/ci-agent-integration.sh", entry.scenario, entry.version, entry.image ?? ""], { cwd: root, env, encoding: "utf8" }); + assert.equal(result.status, 0, result.stderr); + const invocations = commands(); + if (entry.scenario === "rocketmq") { + assert.ok(invocations.some((call) => call.tool === "bash" && call.args.at(-1) === entry.version)); + return; + } + const start = invocations.find((call) => call.tool === "docker" && call.args[0] === "run"); + assert.ok(start, "starts the selected container"); + const expectedImage = entry.image ?? (entry.scenario.startsWith("zookeeper") ? `zookeeper:${entry.version}` + : entry.scenario === "rabbitmq" ? `rabbitmq:${entry.version}-management` : `cassandra:${entry.version}`); + assert.equal(start.args.at(-1), expectedImage); + const last = invocations.at(-1); + assert.equal(last.tool, "docker"); + assert.deepEqual(last.args.slice(0, 2), ["rm", "-fv"]); + const tests = invocations.filter((call) => ["go", "cargo"].includes(call.tool)); + assert.ok(tests.length); + if (entry.scenario === "zookeeper") { + assert.equal(tests.length, entry.version === "3.7.0" ? 2 : 1); + assert.ok(tests[0].args.includes("^TestZooKeeperIntegration$")); + if (entry.version === "3.7.0") assert.ok(tests[1].args.includes("^TestZooKeeperLargeChildrenIntegration$")); + } else if (entry.scenario === "zookeeper-sasl") { + assert.equal(tests[0].env.DBX_ZOOKEEPER_TEST_AUTH_SCHEME, "sasl_digest"); + assert.equal(tests[0].env.DBX_ZOOKEEPER_TEST_USERNAME, "dbx"); + assert.ok(start.args.some((arg) => arg.includes("sessionRequireClientSASLAuth=true"))); + } else if (entry.scenario === "tdengine") { + assert.deepEqual(tests[0].args, ["test", "--manifest-path", "drivers/tdengine/Cargo.toml", "--locked", "--test", "live", "--", "--nocapture"]); + assert.equal(tests[0].env.TDENGINE_INTEGRATION, "1"); + assert.equal(tests[0].env.TDENGINE_TEST_PORT, "6041"); + } else if (entry.scenario === "cassandra") assert.ok(tests[0].args.includes("^TestCassandraIntegration$")); + else { + assert.ok(tests[0].args.includes("^TestRabbitMQIntegration$")); + assert.equal(tests[0].env.RABBITMQ_INTEGRATION, "1"); + assert.ok(start.args.includes("999:999")); + assert.ok(start.args.includes("/var/lib/rabbitmq:rw,exec,uid=999,gid=999,mode=700")); + } + })); +} + +test("failed live tests stay failed and still clean their container", () => withToolStubs((env, commands) => { + const result = spawnSync("/bin/bash", [".github/scripts/ci-agent-integration.sh", "cassandra", "5.0.6"], { cwd: root, env, encoding: "utf8" }); + assert.equal(result.status, 23); + assert.deepEqual(commands().at(-1).args.slice(0, 2), ["rm", "-fv"]); +}, { CI_STUB_FAIL: "go" })); + +test("lock preflight resolves full graphs, keeps --locked and propagates failures", () => { + const plan = { rust: true, agent_rust: { include: [{ driver: "duckdb" }, { driver: "tdengine" }] } }; + assert.deepEqual(lockfileManifests(plan), ["Cargo.toml", "agents/drivers/duckdb/Cargo.toml", "agents/drivers/tdengine/Cargo.toml"]); + assert.throws(() => lockfileManifests({ rust: false, agent_rust: { include: [{ driver: "../../unknown" }] } })); + for (const failure of ["", "cargo"]) withToolStubs((env, commands) => { + const result = spawnSync(process.execPath, [".github/scripts/ci-lockfiles.mjs"], { cwd: root, env, encoding: "utf8" }); + assert.equal(result.status, failure ? 23 : 0, result.stderr); + assert.equal(commands().length, failure ? 1 : 3); + for (const command of commands()) { + assert.equal(command.args[0], "metadata"); + assert.ok(command.args.includes("--locked")); + assert.ok(!command.args.includes("--no-deps")); + } + }, { CI_PLAN: JSON.stringify(plan), CI_STUB_FAIL: failure }); +}); diff --git a/.github/scripts/ci-gate.mjs b/.github/scripts/ci-gate.mjs new file mode 100644 index 0000000000..46ee5891e5 --- /dev/null +++ b/.github/scripts/ci-gate.mjs @@ -0,0 +1,40 @@ +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +export function gateFailures(needs, mode) { + if (needs.changes?.result !== "success") return ["changes did not succeed"]; + let plan; + try { plan = JSON.parse(needs.changes.outputs.plan); } catch { return ["missing or invalid CI plan"]; } + if (!plan || typeof plan !== "object" || Array.isArray(plan)) return ["incomplete CI plan"]; + const flags = ["rust", "agents", "fast", "agent_java", "agent_go_changed", "agent_rust_changed", "agent_integration_changed"]; + if (!flags.every((flag) => typeof plan[flag] === "boolean")) return ["incomplete CI plan"]; + const routedJobs = { frontend: "frontend", packages: "packages", "github-scripts": "github_scripts", + "windows-win7-bundle": "windows_win7_bundle", "duckdb-windows-driver": "duckdb_windows", + jdbc: "jdbc", "offline-jdbc-release": "offline_jdbc", "nix-packaging": "nix" }; + if (mode === "all" && !Object.values(routedJobs).every((output) => ["true", "false"].includes(needs.changes.outputs[output]))) { + return ["missing or invalid job selection outputs"]; + } + const expected = mode === "rust" ? { + "fast-checks": plan.fast, "rust-fmt-clippy": plan.rust, "rust-test": plan.rust, + } : mode === "agents" ? { + "fast-checks": plan.fast, "agent-checks": plan.agents, "agent-java": plan.agent_java, "agent-go": plan.agent_go_changed, + "agent-rust": plan.agent_rust_changed, "agent-integration": plan.agent_integration_changed, + } : mode === "all" ? { + rust: true, agents: true, "fast-checks": plan.fast, + ...Object.fromEntries(Object.entries(routedJobs) + .map(([job, output]) => [job, needs.changes.outputs[output] === "true"])), + } : null; + if (!expected) return [`unknown gate: ${mode}`]; + return Object.entries(expected).flatMap(([job, required]) => { + const result = needs[job]?.result; + return result === "success" || (!required && result === "skipped") ? [] : [`${job}: ${result ?? "missing"} (required=${required})`]; + }); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + const failures = gateFailures(JSON.parse(process.env.NEEDS_JSON), process.argv[2]); + if (failures.length) { + console.error(failures.join("\n")); + process.exitCode = 1; + } else console.log("All selected CI jobs succeeded; only unselected jobs may be skipped."); +} diff --git a/.github/scripts/ci-guide.md b/.github/scripts/ci-guide.md new file mode 100644 index 0000000000..d393757471 --- /dev/null +++ b/.github/scripts/ci-guide.md @@ -0,0 +1,108 @@ +# CI job boundaries + +Paths in this guide are relative to `.github/`. + +`workflows/ci.yml` retains one workflow and one cancellation group. Splitting jobs +does not change release workflows or branch-protection settings. The existing +`rust` and `agents` aggregate check names remain; `ci` additionally summarizes all +selected jobs. None of these gates accepts a failed, cancelled, missing, or +unexpectedly skipped prerequisite. + +## Selection + +`scripts/ci-plan.mjs` reads Cargo workspace metadata and the exact event base/head +diff. Renames include both paths and deleted paths remain visible. Rust package +selection follows reverse dependencies, including build and test dependencies. +Connection-type descriptors belong to `dbx-types`; dialect descriptors belong to +`dbx-sql`. A changed core dependency also selects the standalone DuckDB Agent tests, +whose development dependency is `dbx-core`. + +- Ordinary Rust PRs select foundation, driver, and application test groups from + `scripts/ci-config.mjs`. Each group contains several packages, not one runner per + crate. At most three Rust test groups run concurrently. +- Main pushes, shared Cargo manifests/locks, toolchain/config/vendor changes, CI + infrastructure changes, an unknown workspace member, or an unavailable diff + select the original full-workspace test lane instead of duplicating it with all + three groups. +- Strict clippy still checks the complete workspace and all targets. The existing + full/fast feature distinction is retained: fast skips system-font discovery, + not the other previously enabled capabilities. +- Native Agent source changes select the affected native driver and its live + cases. Shared Agent inputs conservatively select all existing Agent checks. + Windows DuckDB packaging is selected by its own inputs or shared build/lock + changes, not every change to its development dependency's source. +- Existing frontend, package, Windows, JDBC, offline-payload and Nix jobs retain + their commands and path filters. + +Changes to the planner itself select the broad checks. Missing/invalid planning +data fails the aggregate gates rather than producing a successful skip. + +## Fast checks and Rust feature coverage + +`fast-checks` runs formatting and CI/architecture contracts before dependency +resolution or compilation. It checks generated connection types without rewriting +them and validates the relevant root and independent Agent Cargo lockfiles with +full `cargo metadata --locked`; `--no-deps` is intentionally not used for that +validation. + +Cargo may unify features supplied by different workspace consumers. Grouped tests +therefore explicitly retain the features previously enabled on each package, +including plugin runtime defaults and the driver's bundled SQLite capability. +`scripts/ci-rust-coverage.mjs` compares each selected workspace package's enabled +features with the full-workspace invocation and verifies that the groups cover +every workspace member exactly once. It fails if a selected package loses a +feature. An unassigned new workspace member forces the full-workspace lane and +bypasses only the group-specific audit until it is assigned a group. This does not substitute for +the full-workspace integration lane, and does not use `--all-features` to combine +incompatible SQLite backends. + +## Agent jobs + +| Job | Coverage | Maximum parallelism | +| --- | --- | --- | +| `agent-checks` | Existing Python/script tests and Agent source validation | 1 | +| `agent-java` | Existing Java tests, `shadowJar`, and JAR validation | 1 | +| `agent-rust` | DuckDB and TDengine tests; TDengine release build | 2 | +| `agent-go` | Ten existing native Go test/build targets, original race flags, six ZooKeeper cross-build targets | 4 | +| `agent-integration` | Sixteen existing engine/version/authentication cases | 4 | + +Matrix jobs set `fail-fast: false`, so one failure does not conceal results from +the other selected cases. Live test predicates, readiness checks, time limits, +credentials for disposable fixtures, and failure cleanup are retained. +Native, Java and live jobs wait for fast checks and Agent source validation before +starting expensive builds or containers. +RocketMQ's existing integration harness starts official server JARs through Maven +and Java rather than Docker; those matrix entries explicitly retain Java 21. + +| Integration scenario | Versions | +| --- | --- | +| ZooKeeper | 3.4.14, 3.5.5, 3.7.0, 3.9.5; large-child-list coverage remains on 3.7.0 | +| ZooKeeper SASL DIGEST-MD5 | 3.7.0 | +| RocketMQ | 4.9.8, 5.3.1 | +| TDengine | 2.4.0.14, 2.6.0.34, 3.0.7.1, 3.3.6.13, 3.4.2.2; the last uses `tdengine/tsdb` | +| Cassandra | 3.11.19, 5.0.6 | +| RabbitMQ | 3.13, 4.3 | + +## Validation + +Run these from the repository root: + +```sh +node --test .github/scripts/ci-*.test.mjs +node .github/scripts/ci-rust-coverage.mjs +actionlint .github/workflows/ci.yml +bash -n .github/scripts/ci-agent-go.sh +bash -n .github/scripts/ci-agent-integration.sh +``` + +The shell-command tests replace Docker, compilers, readiness probes and delegated +integration scripts with recording stubs. They verify all configured cases and +failure propagation without starting an engine; they are not live database tests. +Do not run the integration shell script on a developer machine. It targets +disposable Actions runners; equivalent validation on the designated test server +must follow that server's isolated-container naming and port policies. + +The split aims to isolate failures, return small-check results earlier, and remove +unrelated work. Rust groups can still repeat dependency compilation across runners; +compare cold/warm build, link, test and queue time on actual runs before claiming a +speedup or changing group sizes. Cancelled runs are not successful benchmarks. diff --git a/.github/scripts/ci-lockfiles.mjs b/.github/scripts/ci-lockfiles.mjs new file mode 100644 index 0000000000..b15b786aa4 --- /dev/null +++ b/.github/scripts/ci-lockfiles.mjs @@ -0,0 +1,23 @@ +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +export function lockfileManifests(plan) { + return [...(plan.rust ? ["Cargo.toml"] : []), + ...plan.agent_rust.include.map(({ driver }) => { + if (!["duckdb", "tdengine"].includes(driver)) throw new Error(`Unknown native Rust driver: ${driver}`); + return `agents/drivers/${driver}/Cargo.toml`; + })]; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + const plan = JSON.parse(process.env.CI_PLAN); + for (const manifest of lockfileManifests(plan)) { + console.log(`Checking locked dependency resolution: ${manifest}`); + const result = spawnSync("cargo", ["metadata", "--locked", "--manifest-path", manifest, "--format-version", "1"], { + stdio: ["ignore", "ignore", "inherit"], + }); + if (result.error) throw result.error; + if (result.status !== 0) process.exit(result.status ?? 1); + } +} diff --git a/.github/scripts/ci-plan.mjs b/.github/scripts/ci-plan.mjs new file mode 100644 index 0000000000..b97cfad679 --- /dev/null +++ b/.github/scripts/ci-plan.mjs @@ -0,0 +1,93 @@ +import { appendFileSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { goAgents, integrationCases, rustAgents, rustGroups } from "./ci-config.mjs"; + +export function changedPaths(base, root) { + if (!base || /^0+$/.test(base)) return null; + execFileSync("git", ["rev-parse", "--verify", `${base}^{commit}`], { cwd: root, stdio: "pipe" }); + return execFileSync("git", ["diff", "--no-renames", "--name-only", "-z", base, "HEAD", "--"], { + cwd: root, encoding: "utf8", maxBuffer: 16 * 1024 * 1024, + }).split("\0").filter(Boolean); +} + +export function planCi({ files, metadata, root, eventName = "pull_request", rustChanged = false, agentsChanged = false }) { + const unknownDiff = files === null; + files ??= []; + const packages = metadata.packages.filter((pkg) => metadata.workspace_members.includes(pkg.id)); + const members = new Map(packages.map((pkg) => [pkg.name, pkg])); + const packagePaths = packages.map((pkg) => [pkg.name, `${path.relative(root, path.dirname(pkg.manifest_path)).split(path.sep).join("/")}/`]); + const ciChanged = unknownDiff || files.some((file) => /^\.github\/(?:workflows\/ci(?:[.-]|\/)|scripts\/ci-|actions\/ci-)/.test(file)); + const sharedRust = ciChanged || files.some((file) => /^(?:Cargo\.(?:toml|lock)$|rust-toolchain|\.cargo\/|vendor\/)/.test(file) + || /^(?:crates\/[^/]+|src-tauri)\/Cargo\.toml$/.test(file)); + const affected = new Set(); + let unknownRust = false; + for (const file of files) { + const owner = packagePaths.find(([, directory]) => file.startsWith(directory)); + if (owner) affected.add(owner[0]); + else if (/^(?:crates|src-tauri)\//.test(file)) unknownRust = true; + if (file.startsWith("plugins/connection-types/")) affected.add("dbx-types"); + if (file.startsWith("plugins/dialects/")) affected.add("dbx-sql"); + } + const knownGroups = new Set(Object.values(rustGroups).flat()); + const unknownMember = packages.some((pkg) => !knownGroups.has(pkg.name)); + const rust = rustChanged || sharedRust || unknownRust || affected.size > 0 || files.includes("scripts/core-architecture.test.mjs"); + const full = rust && (eventName !== "pull_request" || sharedRust || unknownRust || unknownMember || affected.size === 0); + if (full) for (const pkg of packages) affected.add(pkg.name); + let expanded; + do { + expanded = false; + for (const pkg of packages) { + if (!affected.has(pkg.name) && pkg.dependencies.some((dependency) => members.has(dependency.name) && affected.has(dependency.name))) { + affected.add(pkg.name); + expanded = true; + } + } + } while (expanded); + const rustMatrix = !rust ? [] : full ? [{ group: "workspace" }] : Object.entries(rustGroups) + .filter(([, names]) => names.some((name) => affected.has(name))) + .map(([group]) => ({ group })); + const nativeDrivers = [...goAgents.map((agent) => agent.driver), ...rustAgents]; + const nativeChanges = new Set(nativeDrivers.filter((driver) => files.some((file) => file.startsWith(`agents/drivers/${driver}/`)))); + const sharedAgents = ciChanged || sharedRust || files.some((file) => file.startsWith("agents/") + && !nativeDrivers.some((driver) => file.startsWith(`agents/drivers/${driver}/`))) + || files.some((file) => file.startsWith(".github/scripts/bump-agent-versions.") || file === ".github/workflows/agents-release.yml" + || file === "crates/dbx-drivers/assets/agent-protocol-v2.json"); + const allAgents = sharedAgents || (agentsChanged && nativeChanges.size === 0); + if (allAgents) { + for (const driver of nativeDrivers) nativeChanges.add(driver); + } + if (affected.has("dbx-core")) nativeChanges.add("duckdb"); + const goMatrix = goAgents.filter((agent) => nativeChanges.has(agent.driver)); + const nativeRustMatrix = rustAgents.filter((driver) => nativeChanges.has(driver)).map((driver) => ({ driver })); + const liveMatrix = integrationCases.filter((entry) => nativeChanges.has(entry.driver)); + const java = allAgents; + const agents = agentsChanged || sharedAgents || nativeChanges.size > 0; + return { + rust, rust_full: full, rust_matrix: { include: rustMatrix }, rust_groups_known: !unknownMember, + affected_packages: [...affected].sort(), + agents, agent_java: java, agent_go: { include: goMatrix }, agent_rust: { include: nativeRustMatrix }, + agent_integration: { include: liveMatrix }, agent_go_changed: goMatrix.length > 0, + agent_rust_changed: nativeRustMatrix.length > 0, agent_integration_changed: liveMatrix.length > 0, + duckdb_changed: nativeChanges.has("duckdb"), + duckdb_windows: sharedRust || files.some((file) => file.startsWith("agents/drivers/duckdb/")), + fast: rust || agents || ciChanged, + }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + const root = process.cwd(); + const metadata = JSON.parse(execFileSync("cargo", ["metadata", "--locked", "--offline", "--no-deps", "--format-version", "1"], { + cwd: root, encoding: "utf8", maxBuffer: 16 * 1024 * 1024, + })); + const plan = planCi({ + files: changedPaths(process.env.BASE_SHA, root), metadata, root, eventName: process.env.GITHUB_EVENT_NAME, + rustChanged: process.env.RUST_CHANGED === "true", agentsChanged: process.env.AGENTS_CHANGED === "true", + }); + if (process.env.GITHUB_OUTPUT) { + const outputs = { ...plan, plan }; + appendFileSync(process.env.GITHUB_OUTPUT, Object.entries(outputs).map(([name, value]) => `${name}=${JSON.stringify(value)}\n`).join("")); + } + console.log(JSON.stringify(plan, null, 2)); +} diff --git a/.github/scripts/ci-plan.test.mjs b/.github/scripts/ci-plan.test.mjs new file mode 100644 index 0000000000..6b6166d8de --- /dev/null +++ b/.github/scripts/ci-plan.test.mjs @@ -0,0 +1,232 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import { goAgents, integrationCases, rustGroups } from "./ci-config.mjs"; +import { changedPaths, planCi } from "./ci-plan.mjs"; +import { rustCommand } from "./ci-rust.mjs"; +import { gateFailures } from "./ci-gate.mjs"; +import { assertCoverage, parseCoverage } from "./ci-rust-coverage.mjs"; + +const root = "/fixture"; +const graph = { + "dbx-types": [], "dbx-platform": [], "dbx-formats": [], "dbx-sql": ["dbx-types"], + "dbx-ai-provider": ["dbx-platform"], "dbx-plugin-runtime": ["dbx-platform", "dbx-types"], + "dbx-sqlite-worker": [], "dbx-drivers": ["dbx-platform", "dbx-sql", "dbx-types", "dbx-sqlite-worker"], + "dbx-core": ["dbx-drivers", "dbx-sql", "dbx-types", "dbx-platform", "dbx-ai-provider", "dbx-plugin-runtime", "dbx-formats"], + "dbx-mcp": ["dbx-core"], "dbx-cli": ["dbx-core", "dbx-mcp"], "dbx-web": ["dbx-core", "dbx-mcp"], "dbx": ["dbx-core", "dbx-mcp"], +}; +const metadata = { + workspace_members: Object.keys(graph), + packages: Object.entries(graph).map(([name, dependencies]) => ({ + id: name, name, manifest_path: `${root}/${name === "dbx" ? "src-tauri" : `crates/${name}`}/Cargo.toml`, + dependencies: dependencies.map((dependency) => ({ name: dependency })), + })), +}; +const plan = (files, options = {}) => planCi({ files, metadata, root, ...options }); +const groups = (result) => result.rust_matrix.include.map((entry) => entry.group); + +test("foundation changes select transitive consumers and standalone DuckDB", () => { + const result = plan(["crates/dbx-types/src/lib.rs"]); + assert.deepEqual(groups(result), ["foundation", "drivers", "application"]); + for (const name of ["dbx-types", "dbx-sql", "dbx-drivers", "dbx-core", "dbx-web", "dbx-cli", "dbx-mcp", "dbx"]) { + assert.ok(result.affected_packages.includes(name), name); + } + assert.deepEqual(result.agent_rust.include, [{ driver: "duckdb" }]); + assert.equal(result.duckdb_changed, true); + assert.equal(result.duckdb_windows, false); + assert.equal(result.rust_groups_known, true); + assert.equal(result.agent_java, false); +}); + +test("core and consumer changes do not pull unrelated foundation test groups", () => { + assert.deepEqual(groups(plan(["crates/dbx-core/src/query/mod.rs"])), ["application"]); + const cli = plan(["crates/dbx-cli/src/main.rs"]); + assert.deepEqual(groups(cli), ["application"]); + assert.equal(cli.agents, false); + assert.deepEqual(groups(plan(["crates/dbx-drivers/src/db/postgres.rs"])), ["drivers", "application"]); +}); + +test("code generation and test-only dependencies participate in impact analysis", () => { + assert.deepEqual(groups(plan(["plugins/connection-types/postgres.yaml"])), ["foundation", "drivers", "application"]); + assert.deepEqual(groups(plan(["plugins/dialects/postgres.yaml"])), ["foundation", "drivers", "application"]); + const fixture = structuredClone(metadata); + fixture.packages.find((pkg) => pkg.name === "dbx-types").dependencies.push({ name: "dbx-formats", kind: "dev" }); + assert.ok(plan(["crates/dbx-formats/src/lib.rs"], { metadata: fixture }).affected_packages.includes("dbx-types")); +}); + +for (const file of ["Cargo.toml", "Cargo.lock", ".cargo/config.toml", "rust-toolchain.toml", "vendor/wry/src/lib.rs", + "crates/dbx-sql/Cargo.toml", ".github/workflows/ci.yml", ".github/scripts/ci-plan.mjs", ".github/scripts/ci-gate.test.mjs"]) { + test(`shared input ${file} retains full workspace and Agent coverage`, () => { + const result = plan([file]); + assert.deepEqual(groups(result), ["workspace"]); + assert.equal(result.rust_full, true); + assert.equal(result.agent_go.include.length, 10); + assert.equal(result.agent_rust.include.length, 2); + assert.equal(result.agent_integration.include.length, 16); + assert.equal(result.agent_java, true); + assert.equal(result.duckdb_windows, true); + }); +} + +test("main and unknown graph changes fail open to full test coverage", () => { + assert.deepEqual(groups(plan(["crates/dbx-core/src/lib.rs"], { eventName: "push" })), ["workspace"]); + assert.deepEqual(groups(plan(["crates/new-engine/src/lib.rs"])), ["workspace"]); + assert.deepEqual(groups(plan(null)), ["workspace"]); + const fixture = structuredClone(metadata); + fixture.packages.push({ name: "new-member", id: "new-member", manifest_path: `${root}/crates/new-member/Cargo.toml`, dependencies: [] }); + fixture.workspace_members.push("new-member"); + assert.deepEqual(groups(plan(["crates/dbx-core/src/lib.rs"], { metadata: fixture })), ["workspace"]); + assert.equal(plan(["crates/dbx-core/src/lib.rs"], { metadata: fixture }).rust_groups_known, false); + assert.deepEqual(groups(plan(["scripts/core-architecture.test.mjs"])), ["workspace"]); +}); + +test("unrelated docs skip Rust and Agents without inventing a matrix row", () => { + const result = plan(["docs/README.md"]); + assert.equal(result.rust, false); + assert.equal(result.agents, false); + assert.equal(result.fast, false); + assert.deepEqual(result.rust_matrix, { include: [] }); +}); + +for (const driver of ["duckdb", "tdengine", ...goAgents.map((entry) => entry.driver)]) { + test(`native ${driver} changes only select their Agent tests and live cases`, () => { + const result = plan([`agents/drivers/${driver}/source`], { agentsChanged: true }); + assert.equal(result.rust, false); + assert.equal(result.agent_java, false); + assert.ok([...result.agent_go.include, ...result.agent_rust.include].every((entry) => entry.driver === driver)); + assert.ok(result.agent_integration.include.every((entry) => entry.driver === driver)); + }); +} + +test("shared Agent inputs and unknown native modules never silently lose coverage", () => { + for (const file of ["agents/common/src/main/java/Protocol.java", "agents/scripts/validate_agents.py", "agents/build.gradle", + "agents/drivers/new-driver/main.go", "crates/dbx-drivers/assets/agent-protocol-v2.json", ".github/workflows/agents-release.yml"]) { + const result = plan([file]); + assert.equal(result.agent_go.include.length, 10, file); + assert.equal(result.agent_integration.include.length, 16, file); + assert.equal(result.agent_java, true, file); + } + const fallback = plan(["future-agent-filter-input"], { agentsChanged: true }); + assert.equal(fallback.agent_go.include.length, 10); + assert.equal(fallback.agent_integration.include.length, 16); + assert.equal(fallback.agent_java, true); +}); + +test("the independent live matrix preserves every old version and authentication case", () => { + assert.deepEqual(integrationCases.map((entry) => `${entry.scenario}:${entry.version}`), [ + "zookeeper:3.4.14", "zookeeper:3.5.5", "zookeeper:3.7.0", "zookeeper:3.9.5", "zookeeper-sasl:3.7.0", + "rocketmq:4.9.8", "rocketmq:5.3.1", "tdengine:2.4.0.14", "tdengine:2.6.0.34", "tdengine:3.0.7.1", + "tdengine:3.3.6.13", "tdengine:3.4.2.2", "cassandra:3.11.19", "cassandra:5.0.6", "rabbitmq:3.13", "rabbitmq:4.3", + ]); + assert.equal(integrationCases.find((entry) => entry.version === "3.4.2.2").image, "tdengine/tsdb:3.4.2.2"); + assert.deepEqual(goAgents.filter((entry) => entry.race).map((entry) => entry.driver), ["rocketmq", "zookeeper"]); +}); + +test("test groups partition every current workspace package exactly once", () => { + const selected = Object.values(rustGroups).flat(); + assert.equal(new Set(selected).size, selected.length); + assert.deepEqual(selected.toSorted(), Object.keys(graph).toSorted()); +}); + +test("workspace Cargo flags preserve full versus fast coverage and strict clippy", () => { + for (const mode of ["fast", "full"]) { + const command = rustCommand("test", "workspace", mode); + const features = command.at(-1).split(","); + assert.ok(command.includes("--workspace")); + assert.ok(command.includes("--locked")); + for (const pkg of ["dbx", "dbx-core", "dbx-web"]) { + for (const feature of ["duckdb-sidecar", "dynamodb", "mq-admin", "sqlite-sqlcipher"]) assert.ok(features.includes(`${pkg}/${feature}`)); + assert.equal(features.includes(`${pkg}/system-fonts`), mode === "full"); + } + assert.deepEqual(rustCommand("clippy", "workspace", mode).slice(-3), ["--", "-D", "warnings"]); + } + for (const group of Object.keys(rustGroups)) assert.ok(rustCommand("test", group, "fast").includes("--package")); + assert.throws(() => rustCommand("test", "unknown", "full")); + assert.throws(() => rustCommand("clippy", "foundation", "full")); +}); + +function results(files) { + const selected = plan(files); + return { changes: { result: "success", outputs: { plan: JSON.stringify(selected) } }, + "fast-checks": { result: selected.fast ? "success" : "skipped" }, + "rust-fmt-clippy": { result: selected.rust ? "success" : "skipped" }, "rust-test": { result: selected.rust ? "success" : "skipped" }, + "agent-checks": { result: selected.agents ? "success" : "skipped" }, "agent-java": { result: selected.agent_java ? "success" : "skipped" }, + "agent-go": { result: selected.agent_go_changed ? "success" : "skipped" }, + "agent-rust": { result: selected.agent_rust_changed ? "success" : "skipped" }, + "agent-integration": { result: selected.agent_integration_changed ? "success" : "skipped" } }; +} + +test("gates accept only successful selected jobs and intentionally skipped unselected jobs", () => { + for (const files of [["docs/README.md"], ["Cargo.lock"], ["agents/drivers/rabbitmq/main.go"]]) { + for (const mode of ["rust", "agents"]) assert.deepEqual(gateFailures(results(files), mode), []); + } + for (const result of ["skipped", "failure", "cancelled", undefined]) { + const needs = results(["Cargo.lock"]); + needs["rust-test"].result = result; + assert.ok(gateFailures(needs, "rust").length); + needs["agent-integration"].result = result; + assert.ok(gateFailures(needs, "agents").length); + } + const needs = results(["docs/README.md"]); + needs["agent-go"].result = "cancelled"; + assert.ok(gateFailures(needs, "agents").length); + needs.changes.result = "failure"; + assert.ok(gateFailures(needs, "rust").length); + assert.ok(gateFailures({ changes: { result: "success", outputs: { plan: "{}" } } }, "rust").length); + assert.ok(gateFailures({ changes: { result: "success", outputs: { plan: "null" } } }, "rust").length); +}); + +test("git diff routing includes both sides of renames, deleted files, and unusual filenames", () => { + const directory = mkdtempSync(path.join(tmpdir(), "dbx-ci-plan-")); + const git = (...args) => execFileSync("git", args, { cwd: directory, encoding: "utf8", stdio: "pipe" }).trim(); + try { + git("init"); git("config", "user.email", "fixture@example.invalid"); git("config", "user.name", "CI Fixture"); + writeFileSync(path.join(directory, "old.rs"), "unchanged source\n"); + writeFileSync(path.join(directory, "deleted.rs"), "gone\n"); + git("add", "."); git("-c", "core.hooksPath=/dev/null", "commit", "-m", "fixture baseline"); + const base = git("rev-parse", "HEAD"); + git("mv", "old.rs", "renamed\nfile.rs"); git("rm", "deleted.rs"); + git("-c", "core.hooksPath=/dev/null", "commit", "-m", "fixture change"); + assert.deepEqual(changedPaths(base, directory).toSorted(), ["old.rs", "deleted.rs", "renamed\nfile.rs"].toSorted()); + assert.equal(changedPaths("0".repeat(40), directory), null); + assert.throws(() => changedPaths("not-a-ref", directory)); + } finally { rmSync(directory, { recursive: true, force: true }); } +}); + +test("the final gate rejects absent routing outputs and skipped selected jobs", () => { + const needs = results(["docs/README.md"]); + const routed = { frontend: "frontend", packages: "packages", "github-scripts": "github_scripts", + "windows-win7-bundle": "windows_win7_bundle", "duckdb-windows-driver": "duckdb_windows", jdbc: "jdbc", + "offline-jdbc-release": "offline_jdbc", "nix-packaging": "nix" }; + needs.rust = needs.agents = { result: "success" }; + for (const [job, output] of Object.entries(routed)) { + needs[job] = { result: "skipped" }; + needs.changes.outputs[output] = "false"; + } + assert.deepEqual(gateFailures(needs, "all"), []); + needs.changes.outputs.frontend = "true"; + assert.ok(gateFailures(needs, "all").length); + needs.frontend.result = "success"; + assert.deepEqual(gateFailures(needs, "all"), []); + delete needs.changes.outputs.nix; + assert.ok(gateFailures(needs, "all").length); +}); + +test("coverage audit rejects lost features, omitted packages, and overlapping group selection", () => { + const coverage = { workspace: Object.fromEntries(Object.keys(graph).map((name) => [name, ["fixture"]])), + ...Object.fromEntries(Object.entries(rustGroups).map(([group, packages]) => [group, Object.fromEntries(packages.map((name) => [name, ["fixture"]]))])) }; + assertCoverage(coverage); + const lost = structuredClone(coverage); + lost.foundation["dbx-types"] = []; + assert.throws(() => assertCoverage(lost), /lost workspace features/); + const missing = structuredClone(coverage); + delete missing.drivers["dbx-drivers"]; + assert.throws(() => assertCoverage(missing), /wrong packages/); + const overlap = structuredClone(coverage); + overlap.foundation["dbx-core"] = ["fixture"]; + assert.throws(() => assertCoverage(overlap), /wrong packages/); + assert.deepEqual(parseCoverage("dbx-core v0.1.0 (/fixture)|default,openapi\n"), { "dbx-core": ["default", "openapi"] }); +}); diff --git a/.github/scripts/ci-rust-coverage.mjs b/.github/scripts/ci-rust-coverage.mjs new file mode 100644 index 0000000000..149c1e7366 --- /dev/null +++ b/.github/scripts/ci-rust-coverage.mjs @@ -0,0 +1,39 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { rustGroups } from "./ci-config.mjs"; +import { rustCommand } from "./ci-rust.mjs"; + +export function parseCoverage(output) { + return Object.fromEntries(output.trim().split("\n").filter(Boolean).map((line) => { + const [name, features] = line.split("|"); + assert.ok(features !== undefined, `Invalid Cargo tree output: ${line}`); + return [name.split(" ")[0], features.split(",").filter(Boolean)]; + })); +} + +export function assertCoverage(coverage) { + const names = Object.values(rustGroups).flat(); + assert.equal(new Set(names).size, names.length, "Rust test groups overlap"); + assert.deepEqual(names.toSorted(), Object.keys(coverage.workspace).toSorted(), "Rust test groups must cover every workspace package"); + for (const [group, packages] of Object.entries(rustGroups)) { + assert.deepEqual(Object.keys(coverage[group]).toSorted(), packages.toSorted(), `${group} selected the wrong packages`); + for (const name of packages) { + const missing = coverage.workspace[name].filter((feature) => !coverage[group][name].includes(feature)); + assert.deepEqual(missing, [], `${group}/${name} lost workspace features`); + } + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + const coverage = {}; + for (const group of ["workspace", ...Object.keys(rustGroups)]) { + const args = rustCommand("test", group, "fast"); + args[0] = "tree"; + args.push("--depth", "0", "--prefix", "none", "--format", "{p}|{f}"); + coverage[group] = parseCoverage(execFileSync("cargo", args, { encoding: "utf8", maxBuffer: 8 * 1024 * 1024, stdio: ["ignore", "pipe", "inherit"] })); + } + assertCoverage(coverage); + console.log(JSON.stringify(coverage, null, 2)); +} diff --git a/.github/scripts/ci-rust.mjs b/.github/scripts/ci-rust.mjs new file mode 100644 index 0000000000..9a190f79ca --- /dev/null +++ b/.github/scripts/ci-rust.mjs @@ -0,0 +1,31 @@ +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { rustGroups } from "./ci-config.mjs"; + +export function rustCommand(action, group, mode) { + if (!["test", "clippy"].includes(action) || !["full", "fast"].includes(mode) + || (group !== "workspace" && !Object.hasOwn(rustGroups, group)) || (action === "clippy" && group !== "workspace")) { + throw new Error(`Unknown Rust CI configuration: ${action}/${group}/${mode}`); + } + const capabilityFeatures = ["duckdb-sidecar", "dynamodb", "mq-admin", "sqlite-sqlcipher"]; + if (mode === "full") capabilityFeatures.push("system-fonts"); + const appFeatures = ["dbx", "dbx-core", "dbx-web"].flatMap((name) => capabilityFeatures.map((feature) => `${name}/${feature}`)); + const foundationFeatures = ["dbx-types/mq-admin", "dbx-types/openapi", "dbx-sql/duckdb-sidecar", "dbx-sql/openapi", + "dbx-platform/downloads", "dbx-platform/host-prompts", "dbx-platform/test-support", "dbx-plugin-runtime/default", "dbx-plugin-runtime/test-support"]; + const features = group === "foundation" ? foundationFeatures : group === "drivers" ? [ + ...["duckdb-sidecar", "dynamodb", "mq-admin", "sqlite-bundled", "sqlite-sqlcipher", "test-support"].map((feature) => `dbx-drivers/${feature}`), + "dbx-sqlite-worker/runtime", "dbx-types/openapi", "dbx-sql/openapi", + ] : appFeatures; + const packages = group === "workspace" ? ["--workspace"] : rustGroups[group].flatMap((name) => ["--package", name]); + return [action, ...packages, "--locked", ...(action === "clippy" ? ["--all-targets"] : []), + "--no-default-features", "--features", features.join(","), ...(action === "clippy" ? ["--", "-D", "warnings"] : [])]; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + const args = rustCommand(...process.argv.slice(2, 5)); + console.log(`cargo ${args.join(" ")}`); + const result = spawnSync("cargo", args, { stdio: "inherit" }); + if (result.error) throw result.error; + process.exit(result.status ?? 1); +} diff --git a/.github/scripts/ci-workflow.test.mjs b/.github/scripts/ci-workflow.test.mjs new file mode 100644 index 0000000000..4ff8ebf2e8 --- /dev/null +++ b/.github/scripts/ci-workflow.test.mjs @@ -0,0 +1,73 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +const workflow = readFileSync(new URL("../workflows/ci.yml", import.meta.url), "utf8"); +function job(name) { + const jobs = workflow.slice(workflow.indexOf("\njobs:\n") + 7); + const pattern = /^ ([\w-]+):\s*$/gm; + const definitions = [...jobs.matchAll(pattern)]; + const index = definitions.findIndex((match) => match[1] === name); + assert.ok(index >= 0, `missing job ${name}`); + return jobs.slice(definitions[index].index, definitions[index + 1]?.index ?? jobs.length); +} + +test("fast checks run format and contracts before graph resolution or compilation", () => { + const fast = job("fast-checks"); + for (const command of ["cargo fmt --check", "node --test scripts/core-architecture.test.mjs", + "node scripts/sync-connection-types.mjs --check", "node .github/scripts/ci-lockfiles.mjs", "node .github/scripts/ci-rust-coverage.mjs"]) assert.ok(fast.includes(command)); + assert.ok(fast.indexOf("cargo fmt --check") < fast.indexOf("ci-lockfiles.mjs")); + assert.doesNotMatch(fast, /cargo (?:test|build|clippy)/); + assert.ok(fast.includes("needs.changes.outputs.rust_groups_known == 'true'")); +}); + +test("Agent and Rust matrices are bounded and do not cancel sibling failures", () => { + for (const [name, output, parallel] of [["rust-test", "rust_matrix", 3], ["agent-rust", "agent_rust", 2], + ["agent-go", "agent_go", 4], ["agent-integration", "agent_integration", 4]]) { + const content = job(name); + assert.match(content, /fail-fast: false/); + assert.ok(content.includes(`max-parallel: ${parallel}`)); + assert.ok(content.includes(`fromJSON(needs.changes.outputs.${output})`)); + assert.match(content, /if: needs\.changes\.outputs\.[\w_]+ == 'true'/); + if (name.startsWith("agent-")) assert.ok(content.includes("needs: [changes, fast-checks, agent-checks]")); + } +}); + +test("stable Rust, Agent and overall gates always inspect selected upstream results", () => { + for (const [name, mode, dependencies] of [["rust", "rust", ["fast-checks", "rust-fmt-clippy", "rust-test"]], + ["agents", "agents", ["fast-checks", "agent-checks", "agent-rust", "agent-go", "agent-integration", "agent-java"]], + ["ci", "all", ["rust", "agents", "frontend", "packages", "windows-win7-bundle", "duckdb-windows-driver", "nix-packaging"]]]) { + const content = job(name); + assert.match(content, /if: always\(\)/); + assert.ok(content.includes(`node .github/scripts/ci-gate.mjs ${mode}`)); + assert.ok(content.includes("${{ toJSON(needs) }}")); + for (const dependency of dependencies) assert.match(content, new RegExp(`^ - ${dependency}$`, "m")); + } +}); + +test("every old Agent stage has an independent owner and Java packaging remains strict", () => { + assert.ok(job("agent-checks").includes("python3 -m unittest discover")); + assert.ok(job("agent-checks").includes("python3 scripts/validate_agents.py")); + assert.ok(job("agent-checks").includes("bump-agent-versions.test.mjs")); + assert.ok(job("agent-java").includes("./gradlew test shadowJar --continue")); + assert.ok(job("agent-java").includes("python3 scripts/validate_agent_jars.py")); + assert.ok(job("agent-rust").includes('cargo test --manifest-path "drivers/$DRIVER/Cargo.toml" --locked')); + assert.ok(job("agent-rust").includes("cargo build --manifest-path drivers/tdengine/Cargo.toml --locked --release --bin dbx-tdengine-driver")); + assert.ok(job("agent-go").includes("ci-agent-go.sh")); + assert.ok(job("agent-integration").includes("ci-agent-integration.sh")); + assert.ok(job("agent-integration").includes("Setup RocketMQ server Java")); + assert.ok(job("agent-integration").includes("if: matrix.scenario == 'rocketmq'")); + assert.ok(job("agent-integration").includes('java-version: "21"')); + for (const name of ["agent-rust", "agent-go", "agent-integration", "agent-java"]) assert.doesNotMatch(job(name), /continue-on-error: true/); +}); + +test("the planner uses the exact event base and preserves a single workflow cancellation scope", () => { + const changes = job("changes"); + assert.ok(changes.includes("github.event.pull_request.base.sha || github.event.before")); + assert.ok(changes.includes("node .github/scripts/ci-plan.mjs")); + for (const flag of ["rust", "rust_full", "rust_matrix", "agents", "agent_go", "agent_rust", "agent_integration", "plan"]) { + assert.ok(changes.includes(`steps.plan.outputs.${flag}`)); + } + assert.ok(workflow.includes("group: ${{ github.workflow }}-${{ github.ref }}")); + assert.ok(workflow.includes("cancel-in-progress: true")); +}); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8bfaad5b65..d011a4827c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [main] + branches: [ main ] pull_request: - branches: [main] + branches: [ main ] concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -255,7 +255,9 @@ jobs: "$DRIVER" < /dev/null rust-fmt-clippy: - needs: changes + needs: + - changes + - fast-checks if: needs.changes.outputs.rust == 'true' runs-on: ubuntu-22.04 env: @@ -264,8 +266,6 @@ jobs: RUSTC_WRAPPER: sccache # The fast lane skips only system font discovery while retaining the other default capabilities. RUST_FEATURE_MODE: ${{ github.event_name == 'pull_request' && needs.changes.outputs.rust_full != 'true' && 'fast' || 'full' }} - RUST_FAST_FEATURES: dbx/duckdb-sidecar,dbx/dynamodb,dbx/mq-admin,dbx/sqlite-sqlcipher,dbx-core/duckdb-sidecar,dbx-core/dynamodb,dbx-core/mq-admin,dbx-core/sqlite-sqlcipher,dbx-web/duckdb-sidecar,dbx-web/dynamodb,dbx-web/mq-admin,dbx-web/sqlite-sqlcipher - RUST_FULL_FEATURES: dbx/duckdb-sidecar,dbx/dynamodb,dbx/mq-admin,dbx/sqlite-sqlcipher,dbx/system-fonts,dbx-core/duckdb-sidecar,dbx-core/dynamodb,dbx-core/mq-admin,dbx-core/sqlite-sqlcipher,dbx-core/system-fonts,dbx-web/duckdb-sidecar,dbx-web/dynamodb,dbx-web/mq-admin,dbx-web/sqlite-sqlcipher,dbx-web/system-fonts # Fork PRs cannot read repository secrets, so retain the GHA backend for them. SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }} steps: @@ -324,19 +324,9 @@ jobs: # PR caches are large and branch-scoped; restore them from main without saving per-PR copies. save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Core architecture contracts - run: node --test scripts/core-architecture.test.mjs - - - name: Cargo fmt check - run: cargo fmt --check - - name: Cargo clippy - run: | - if [ "$RUST_FEATURE_MODE" = "fast" ]; then - cargo clippy --workspace --locked --all-targets --no-default-features --features "$RUST_FAST_FEATURES" -- -D warnings - else - cargo clippy --workspace --locked --all-targets --no-default-features --features "$RUST_FULL_FEATURES" -- -D warnings - fi + run: |- + node .github/scripts/ci-rust.mjs clippy workspace "$RUST_FEATURE_MODE" - name: Show sccache stats if: always() @@ -344,7 +334,9 @@ jobs: run: ${SCCACHE_PATH} --show-stats rust-test: - needs: changes + needs: + - changes + - fast-checks if: needs.changes.outputs.rust == 'true' runs-on: ubuntu-22.04 env: @@ -355,10 +347,9 @@ jobs: RUSTC_WRAPPER: sccache # The fast lane skips only system font discovery while retaining the other default capabilities. RUST_FEATURE_MODE: ${{ github.event_name == 'pull_request' && needs.changes.outputs.rust_full != 'true' && 'fast' || 'full' }} - RUST_FAST_FEATURES: dbx/duckdb-sidecar,dbx/dynamodb,dbx/mq-admin,dbx/sqlite-sqlcipher,dbx-core/duckdb-sidecar,dbx-core/dynamodb,dbx-core/mq-admin,dbx-core/sqlite-sqlcipher,dbx-web/duckdb-sidecar,dbx-web/dynamodb,dbx-web/mq-admin,dbx-web/sqlite-sqlcipher - RUST_FULL_FEATURES: dbx/duckdb-sidecar,dbx/dynamodb,dbx/mq-admin,dbx/sqlite-sqlcipher,dbx/system-fonts,dbx-core/duckdb-sidecar,dbx-core/dynamodb,dbx-core/mq-admin,dbx-core/sqlite-sqlcipher,dbx-core/system-fonts,dbx-web/duckdb-sidecar,dbx-web/dynamodb,dbx-web/mq-admin,dbx-web/sqlite-sqlcipher,dbx-web/system-fonts # Fork PRs cannot read repository secrets, so retain the GHA backend for them. SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }} + RUST_TEST_GROUP: ${{ matrix.group }} steps: - uses: actions/checkout@v5 @@ -366,6 +357,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf libssl-dev libsecret-1-dev + if: matrix.group != 'foundation' - name: Setup Rust uses: dtolnay/rust-toolchain@1.97.1 @@ -416,33 +408,37 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - name: Cargo test - run: | - if [ "$RUST_FEATURE_MODE" = "fast" ]; then - cargo test --workspace --locked --no-default-features --features "$RUST_FAST_FEATURES" - else - cargo test --workspace --locked --no-default-features --features "$RUST_FULL_FEATURES" - fi + run: |- + node .github/scripts/ci-rust.mjs test "$RUST_TEST_GROUP" "$RUST_FEATURE_MODE" - name: Show sccache stats if: always() continue-on-error: true run: ${SCCACHE_PATH} --show-stats + strategy: + fail-fast: false + max-parallel: 3 + matrix: ${{ fromJSON(needs.changes.outputs.rust_matrix) }} + name: rust-test (${{ matrix.group }}) rust: - needs: [changes, rust-fmt-clippy, rust-test] - if: always() && needs.changes.outputs.rust == 'true' + needs: + - changes + - fast-checks + - rust-fmt-clippy + - rust-test + if: always() runs-on: ubuntu-22.04 steps: - - name: Check Rust jobs - run: | - if [ "${{ needs.rust-fmt-clippy.result }}" != "success" ]; then - echo "rust-fmt-clippy result: ${{ needs.rust-fmt-clippy.result }}" - exit 1 - fi - if [ "${{ needs.rust-test.result }}" != "success" ]; then - echo "rust-test result: ${{ needs.rust-test.result }}" - exit 1 - fi + - uses: actions/checkout@v5 + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: 22.13.0 + - name: Check selected rust jobs + env: + NEEDS_JSON: ${{ toJSON(needs) }} + run: node .github/scripts/ci-gate.mjs rust jdbc: needs: changes @@ -531,15 +527,26 @@ jobs: outputs: frontend: ${{ steps.filter.outputs.frontend }} packages: ${{ steps.filter.outputs.packages }} - rust: ${{ steps.filter.outputs.rust }} - rust_full: ${{ steps.rust-mode.outputs.full }} + rust: ${{ steps.plan.outputs.rust }} + rust_full: ${{ steps.plan.outputs.rust_full }} jdbc: ${{ steps.filter.outputs.jdbc }} offline_jdbc: ${{ steps.filter.outputs.offline_jdbc }} - agents: ${{ steps.filter.outputs.agents }} - duckdb_windows: ${{ steps.filter.outputs.duckdb_windows }} + agents: ${{ steps.plan.outputs.agents }} + duckdb_windows: ${{ steps.filter.outputs.duckdb_windows == 'true' || steps.plan.outputs.duckdb_windows == 'true' }} nix: ${{ steps.filter.outputs.nix }} windows_win7_bundle: ${{ steps.filter.outputs.windows_win7_bundle }} github_scripts: ${{ steps.filter.outputs.github_scripts }} + fast: ${{ steps.plan.outputs.fast }} + rust_matrix: ${{ steps.plan.outputs.rust_matrix }} + rust_groups_known: ${{ steps.plan.outputs.rust_groups_known }} + agent_java: ${{ steps.plan.outputs.agent_java }} + agent_go: ${{ steps.plan.outputs.agent_go }} + agent_rust: ${{ steps.plan.outputs.agent_rust }} + agent_integration: ${{ steps.plan.outputs.agent_integration }} + agent_go_changed: ${{ steps.plan.outputs.agent_go_changed }} + agent_rust_changed: ${{ steps.plan.outputs.agent_rust_changed }} + agent_integration_changed: ${{ steps.plan.outputs.agent_integration_changed }} + plan: ${{ steps.plan.outputs.plan }} steps: - uses: actions/checkout@v5 with: @@ -659,37 +666,79 @@ jobs: - '.github/workflows/ci.yml' - 'apps/desktop/src/types/database.ts' - 'crates/dbx-core/assets/database-drivers.manifest.json' - - - name: Select Rust feature coverage - id: rust-mode - shell: bash + - name: Setup Rust + uses: dtolnay/rust-toolchain@1.97.1 + - name: Plan dependency-aware CI jobs + id: plan env: BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }} RUST_CHANGED: ${{ steps.filter.outputs.rust }} - run: | - full=false - - if [ "$RUST_CHANGED" = "true" ] && [ "${{ github.event_name }}" = "push" ]; then - full=true - elif [ "$RUST_CHANGED" = "true" ]; then - if [ -z "$BASE_SHA" ] || echo "$BASE_SHA" | grep -Eq '^0+$'; then - BASE_SHA="HEAD~1" - fi - - while IFS= read -r file; do - case "$file" in - Cargo.toml|Cargo.lock|rust-toolchain*|.github/workflows/ci.yml|*/Cargo.toml|vendor/*) - full=true - break - ;; - esac - - done < <(git diff --name-only "$BASE_SHA" HEAD -- Cargo.toml Cargo.lock 'rust-toolchain*' crates src-tauri vendor .github/workflows/ci.yml) - fi - - echo "full=$full" >> "$GITHUB_OUTPUT" + AGENTS_CHANGED: ${{ steps.filter.outputs.agents }} + run: node .github/scripts/ci-plan.mjs agents: + needs: + - changes + - fast-checks + - agent-checks + - agent-java + - agent-rust + - agent-go + - agent-integration + if: always() + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v5 + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: 22.13.0 + - name: Check selected agents jobs + env: + NEEDS_JSON: ${{ toJSON(needs) }} + run: node .github/scripts/ci-gate.mjs agents + fast-checks: + needs: changes + if: needs.changes.outputs.fast == 'true' + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v5 + - name: Setup Rust + uses: dtolnay/rust-toolchain@1.97.1 + with: + components: rustfmt + if: needs.changes.outputs.rust == 'true' || needs.changes.outputs.agent_rust_changed == 'true' + - name: Cargo fmt check + if: needs.changes.outputs.rust == 'true' + run: cargo fmt --check + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: 22.13.0 + - name: CI planner and gate tests + run: node --test .github/scripts/ci-*.test.mjs + - name: Core architecture contracts + if: needs.changes.outputs.rust == 'true' + run: node --test scripts/core-architecture.test.mjs + - name: Setup pnpm + if: needs.changes.outputs.rust == 'true' + uses: pnpm/action-setup@v6 + with: + version: 10.27.0 + - name: Install generator dependencies + if: needs.changes.outputs.rust == 'true' + run: pnpm --filter dbx... install --frozen-lockfile --ignore-scripts + - name: Check generated connection types + if: needs.changes.outputs.rust == 'true' + run: node scripts/sync-connection-types.mjs --check + - name: Check workspace and standalone Cargo locks + env: + CI_PLAN: ${{ needs.changes.outputs.plan }} + run: node .github/scripts/ci-lockfiles.mjs + - name: Check grouped Rust feature coverage + if: needs.changes.outputs.rust == 'true' && needs.changes.outputs.rust_groups_known == 'true' + run: node .github/scripts/ci-rust-coverage.mjs + agent-checks: needs: changes if: needs.changes.outputs.agents == 'true' runs-on: ubuntu-22.04 @@ -698,357 +747,163 @@ jobs: working-directory: agents steps: - uses: actions/checkout@v5 - - - name: Setup Java - uses: actions/setup-java@v5 + - name: Setup Node.js + uses: actions/setup-node@v6 with: - distribution: temurin - java-version: | - 8 - 21 - - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: "1.25.x" - - - name: Setup Rust - uses: dtolnay/rust-toolchain@1.97.1 - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v4 - + node-version: 22.13.0 - name: Install packaging tools run: | sudo apt-get update sudo apt-get install -y zstd - - name: Agent script tests run: | python3 -m unittest discover -s scripts -p '*_test.py' node --test ../.github/scripts/bump-agent-versions.test.mjs - - - name: DuckDB driver tests - run: cargo test --manifest-path drivers/duckdb/Cargo.toml --locked - - - name: TDengine driver tests - run: cargo test --manifest-path drivers/tdengine/Cargo.toml --locked - - name: Agent validation run: python3 scripts/validate_agents.py - - - name: Oracle native agent tests - run: go test ./... - working-directory: agents/drivers/oracle-go - - - name: Xugu native agent tests - run: GONOSUMDB=gitee.com/XuguDB/go-xugu-driver go test ./... - working-directory: agents/drivers/xugu - - - name: RabbitMQ native agent tests - run: go test ./... - working-directory: agents/drivers/rabbitmq - - - name: RocketMQ native agent tests - run: go test -race ./... - working-directory: agents/drivers/rocketmq - - - name: ZooKeeper native agent tests - run: go test -race ./... - working-directory: agents/drivers/zookeeper - - - name: Cassandra native agent tests - run: go test ./... - working-directory: agents/drivers/cassandra-go - - - name: Hive native agent tests - run: go test ./... - working-directory: agents/drivers/hive-go - - - name: Vastbase native agent tests - run: go test ./... - working-directory: agents/drivers/vastbase-go - - - name: Neo4j native agent tests - run: go test ./... - working-directory: agents/drivers/neo4j-go - - - name: IoTDB native agent tests - run: go test ./... - working-directory: agents/drivers/iotdb - - - name: Oracle native agent build - run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-oracle-linux-x64 . - working-directory: agents/drivers/oracle-go - - - name: Xugu native agent build - run: GONOSUMDB=gitee.com/XuguDB/go-xugu-driver CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-xugu-linux-x64 . - working-directory: agents/drivers/xugu - - - name: RabbitMQ native agent build - run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-rabbitmq-linux-x64 . - working-directory: agents/drivers/rabbitmq - - - name: RocketMQ native agent build - run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-rocketmq-linux-x64 . - working-directory: agents/drivers/rocketmq - - - name: ZooKeeper native agent cross-builds - shell: bash - working-directory: agents/drivers/zookeeper + agent-java: + needs: [changes, fast-checks, agent-checks] + if: needs.changes.outputs.agent_java == 'true' + runs-on: ubuntu-22.04 + defaults: + run: + working-directory: agents + steps: + - uses: actions/checkout@v5 + - name: Setup Java + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: |- + 8 + 21 + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + - name: Install packaging tools run: | - set -euo pipefail - for target in darwin/arm64 darwin/amd64 linux/arm64 linux/amd64 windows/arm64 windows/amd64; do - IFS=/ read -r goos goarch <<< "$target" - output="/tmp/dbx-agent-zookeeper-${goos}-${goarch}" - [ "$goos" = windows ] && output="${output}.exe" - CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags="-s -w" -o "$output" . - done - - - name: ZooKeeper native agent integration tests - shell: bash - working-directory: agents/drivers/zookeeper - run: | - set -euo pipefail - for version in 3.4.14 3.5.5 3.7.0 3.9.5; do - name="dbx-zookeeper-${version//./-}" - docker rm -fv "$name" >/dev/null 2>&1 || true - docker run -d --name "$name" -p 2181:2181 "zookeeper:$version" - cleanup() { - docker rm -fv "$name" >/dev/null 2>&1 || true - } - trap cleanup EXIT - ready=false - for _ in $(seq 1 90); do - if timeout 1 bash -c '/dev/null 2>&1; then - ready=true - break - fi - sleep 2 - done - if [ "$ready" != true ]; then - docker logs "$name" - exit 1 - fi - DBX_ZOOKEEPER_TEST_CONNECT_STRING=127.0.0.1:2181 \ - go test -run '^TestZooKeeperIntegration$' -count=1 ./... - if [ "$version" = "3.7.0" ]; then - DBX_ZOOKEEPER_TEST_CONNECT_STRING=127.0.0.1:2181 \ - go test -run '^TestZooKeeperLargeChildrenIntegration$' -count=1 ./... - fi - cleanup - trap - EXIT - done - - - name: ZooKeeper SASL DIGEST-MD5 integration test - shell: bash - working-directory: agents/drivers/zookeeper - run: | - set -euo pipefail - name="dbx-zookeeper-sasl-3-7-0" - jaas_file=$(mktemp) - cat > "$jaas_file" <<'EOF' - Server { - org.apache.zookeeper.server.auth.DigestLoginModule required - user_dbx="dbx-secret"; - }; - EOF - chmod 0644 "$jaas_file" - docker rm -fv "$name" >/dev/null 2>&1 || true - docker run -d --name "$name" -p 2181:2181 \ - -v "$jaas_file:/conf/server-jaas.conf:ro" \ - -e ZOO_CFG_EXTRA="authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider" \ - -e JVMFLAGS="-Djava.security.auth.login.config=/conf/server-jaas.conf -Dzookeeper.sasl.serverconfig=Server -Dzookeeper.sessionRequireClientSASLAuth=true" \ - zookeeper:3.7.0 - cleanup() { - docker rm -fv "$name" >/dev/null 2>&1 || true - } - trap cleanup EXIT - ready=false - for _ in $(seq 1 90); do - if timeout 1 bash -c '/dev/null 2>&1; then - ready=true - break - fi - sleep 2 - done - if [ "$ready" != true ]; then - docker logs "$name" - exit 1 - fi - DBX_ZOOKEEPER_TEST_CONNECT_STRING=127.0.0.1:2181 \ - DBX_ZOOKEEPER_TEST_AUTH_SCHEME=sasl_digest \ - DBX_ZOOKEEPER_TEST_USERNAME=dbx \ - DBX_ZOOKEEPER_TEST_PASSWORD=dbx-secret \ - go test -run '^TestZooKeeperIntegration$' -count=1 ./... - - - name: RocketMQ native agent integration tests - shell: bash - working-directory: agents/drivers/rocketmq - run: | - set -euo pipefail - for version in 4.9.8 5.3.1; do - ./scripts/run-integration.sh "$version" - done - - - name: Cassandra native agent build - run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-cassandra-linux-x64 . - working-directory: agents/drivers/cassandra-go - - - name: Hive native agent build - run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-hive-linux-x64 . - working-directory: agents/drivers/hive-go - - - name: Vastbase native agent build - run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-vastbase-linux-x64 . - working-directory: agents/drivers/vastbase-go - - - name: Neo4j native agent build - run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-neo4j-linux-x64 . - working-directory: agents/drivers/neo4j-go - - - name: IoTDB native agent build - run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-iotdb-linux-x64 . - working-directory: agents/drivers/iotdb - - - name: TDengine native agent build - run: cargo build --manifest-path drivers/tdengine/Cargo.toml --locked --release --bin dbx-tdengine-driver - - - name: TDengine native agent integration tests - shell: bash - run: | - set -euo pipefail - while IFS='|' read -r version image; do - name="dbx-tdengine-${version//./-}" - docker rm -fv "$name" >/dev/null 2>&1 || true - docker run -d --name "$name" \ - --memory 4g \ - --ulimit nofile=65535:65535 \ - -p 6030:6030 \ - -p 6041:6041 \ - "$image" - cleanup() { - docker rm -fv "$name" >/dev/null 2>&1 || true - } - trap cleanup EXIT - ready=false - dnodes="" - for _ in $(seq 1 90); do - dnodes="$(docker exec "$name" taos -s 'SHOW DNODES' 2>/dev/null || true)" - if grep -Eq '\|[[:space:]]*ready[[:space:]]*\|' <<<"$dnodes"; then - ready=true - break - fi - sleep 2 - done - if [ "$ready" != "true" ]; then - printf '%s\n' "$dnodes" - docker logs "$name" - exit 1 - fi - TDENGINE_INTEGRATION=1 \ - TDENGINE_TEST_HOST=127.0.0.1 \ - TDENGINE_TEST_PORT=6041 \ - cargo test --manifest-path drivers/tdengine/Cargo.toml --locked --test live -- --nocapture - cleanup - trap - EXIT - done <<'EOF' - 2.4.0.14|tdengine/tdengine:2.4.0.14 - 2.6.0.34|tdengine/tdengine:2.6.0.34 - 3.0.7.1|tdengine/tdengine:3.0.7.1 - 3.3.6.13|tdengine/tdengine:3.3.6.13 - 3.4.2.2|tdengine/tsdb:3.4.2.2 - EOF - - - name: Cassandra native agent integration tests - shell: bash - working-directory: agents/drivers/cassandra-go - run: | - set -euo pipefail - for version in 3.11.19 5.0.6; do - name="dbx-cassandra-${version//./-}" - docker rm -fv "$name" >/dev/null 2>&1 || true - docker run -d --name "$name" \ - -e CASSANDRA_CLUSTER_NAME="DBX Cassandra CI $version" \ - -e CASSANDRA_DC=dc1 \ - -e CASSANDRA_RACK=rack1 \ - -e CASSANDRA_ENDPOINT_SNITCH=GossipingPropertyFileSnitch \ - -e CASSANDRA_NUM_TOKENS=16 \ - -e MAX_HEAP_SIZE=512M \ - -e HEAP_NEWSIZE=100M \ - -p 9042:9042 \ - "cassandra:$version" - cleanup() { - docker rm -fv "$name" >/dev/null 2>&1 || true - } - trap cleanup EXIT - ready=false - for _ in $(seq 1 100); do - if docker exec "$name" cqlsh -e 'SELECT release_version FROM system.local' >/dev/null 2>&1; then - ready=true - break - fi - sleep 3 - done - if [ "$ready" != "true" ]; then - docker logs "$name" - exit 1 - fi - CASSANDRA_TEST_HOST=127.0.0.1 \ - CASSANDRA_TEST_PORT=9042 \ - go test -run '^TestCassandraIntegration$' -count=1 ./... - cleanup - trap - EXIT - done - - - name: RabbitMQ native agent integration tests - shell: bash - working-directory: agents/drivers/rabbitmq - run: | - set -euo pipefail - for version in 3.13 4.3; do - name="dbx-rabbitmq-${version//./-}" - cookie="dbx-ci-${GITHUB_RUN_ID:-local}-${version//./-}" - # RabbitMQ is disposable in CI. Keep its data on a fresh tmpfs - # owned by the image's rabbitmq user so .erlang.cookie is readable. - docker rm -fv "$name" >/dev/null 2>&1 || true - docker run -d --name "$name" \ - --user 999:999 \ - --tmpfs /var/lib/rabbitmq:rw,exec,uid=999,gid=999,mode=700 \ - -e RABBITMQ_DEFAULT_USER=dbx \ - -e RABBITMQ_DEFAULT_PASS=dbx-password \ - -e RABBITMQ_ERLANG_COOKIE="$cookie" \ - -p 5672:5672 -p 15672:15672 \ - "rabbitmq:${version}-management" - cleanup() { - docker rm -fv "$name" >/dev/null 2>&1 || true - } - trap cleanup EXIT - ready=false - for _ in $(seq 1 60); do - if docker exec "$name" rabbitmq-diagnostics -q check_running >/dev/null 2>&1 \ - && curl --fail --silent --noproxy '*' --user dbx:dbx-password \ - http://127.0.0.1:15672/api/overview >/dev/null; then - ready=true - break - fi - sleep 2 - done - if [ "$ready" != "true" ]; then - docker inspect "$name" --format 'image={{.Config.Image}} user={{.Config.User}} status={{.State.Status}} exit={{.State.ExitCode}}' || true - docker logs "$name" - exit 1 - fi - RABBITMQ_INTEGRATION=1 \ - RABBITMQ_USERNAME=dbx \ - RABBITMQ_PASSWORD=dbx-password \ - go test -run '^TestRabbitMQIntegration$' -count=1 ./... - cleanup - trap - EXIT - done - + sudo apt-get update + sudo apt-get install -y zstd - name: Java agent tests and packages run: ./gradlew test shadowJar --continue - - name: Agent jar validation run: python3 scripts/validate_agent_jars.py + agent-rust: + needs: [changes, fast-checks, agent-checks] + if: needs.changes.outputs.agent_rust_changed == 'true' + runs-on: ubuntu-22.04 + defaults: + run: + working-directory: agents + steps: + - uses: actions/checkout@v5 + - name: Setup Rust + uses: dtolnay/rust-toolchain@1.97.1 + - name: Rust cache + uses: swatinem/rust-cache@v2 + with: + workspaces: agents/drivers/${{ matrix.driver }} -> target + cache-on-failure: true + save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Native Rust driver tests + env: + DRIVER: ${{ matrix.driver }} + run: cargo test --manifest-path "drivers/$DRIVER/Cargo.toml" --locked + - name: TDengine native agent build + if: matrix.driver == 'tdengine' + run: cargo build --manifest-path drivers/tdengine/Cargo.toml --locked --release --bin dbx-tdengine-driver + strategy: + fail-fast: false + max-parallel: 2 + matrix: ${{ fromJSON(needs.changes.outputs.agent_rust) }} + agent-go: + needs: [changes, fast-checks, agent-checks] + if: needs.changes.outputs.agent_go_changed == 'true' + runs-on: ubuntu-22.04 + defaults: + run: + working-directory: agents + steps: + - uses: actions/checkout@v5 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: 1.25.x + cache-dependency-path: agents/drivers/${{ matrix.driver }}/go.sum + - name: Native Go tests and target builds + env: + DRIVER: ${{ matrix.driver }} + BINARY: ${{ matrix.binary }} + RACE: ${{ matrix.race }} + run: bash ../.github/scripts/ci-agent-go.sh "$DRIVER" "$BINARY" "$RACE" + strategy: + fail-fast: false + max-parallel: 4 + matrix: ${{ fromJSON(needs.changes.outputs.agent_go) }} + agent-integration: + needs: [changes, fast-checks, agent-checks] + if: needs.changes.outputs.agent_integration_changed == 'true' + runs-on: ubuntu-22.04 + defaults: + run: + working-directory: agents + steps: + - uses: actions/checkout@v5 + - name: Setup RocketMQ server Java + if: matrix.scenario == 'rocketmq' + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: "21" + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: 1.25.x + cache-dependency-path: agents/drivers/${{ matrix.driver }}/go.sum + if: matrix.driver != 'tdengine' + - name: Setup Rust + uses: dtolnay/rust-toolchain@1.97.1 + if: matrix.driver == 'tdengine' + - name: TDengine Rust cache + if: matrix.driver == 'tdengine' + uses: swatinem/rust-cache@v2 + with: + workspaces: agents/drivers/tdengine -> target + cache-on-failure: true + save-if: ${{ github.ref == 'refs/heads/main' && matrix.version == '3.4.2.2' }} + - name: Run one live engine and version + env: + SCENARIO: ${{ matrix.scenario }} + VERSION: ${{ matrix.version }} + IMAGE: ${{ matrix.image || '' }} + run: bash ../.github/scripts/ci-agent-integration.sh "$SCENARIO" "$VERSION" "$IMAGE" + name: agent-integration (${{ matrix.scenario }}, ${{ matrix.version }}) + strategy: + fail-fast: false + max-parallel: 4 + matrix: ${{ fromJSON(needs.changes.outputs.agent_integration) }} + ci: + needs: + - changes + - fast-checks + - rust + - agents + - frontend + - github-scripts + - packages + - windows-win7-bundle + - duckdb-windows-driver + - jdbc + - offline-jdbc-release + - nix-packaging + if: always() + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v5 + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: 22.13.0 + - name: Check selected all jobs + env: + NEEDS_JSON: ${{ toJSON(needs) }} + run: node .github/scripts/ci-gate.mjs all