github/dbx
4
0
Fork 0
mirror of https://github.com/t8y2/dbx.git synced 2026-10-19 16:16:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ci): split dependency-aware checks and agent matrices

Browse Source
This commit is contained in:
t8y2
2026-09-18 17:06:57 +08:00
parent 23790bd5ee
commit c8966a3de9
13 changed files with 1266 additions and 406 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/ci-agent-go.sh
+21
View File
@@ -0,0 +1,21 @@
#!/usr/bin/env bash
set -euo pipefail
driver="${1:?driver is required}"
binary="${2:?binary name is required}"
race="${3:?race mode is required}"
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
cd "$root/agents/drivers/$driver"
if [ "$driver" = xugu ]; then export GONOSUMDB=gitee.com/XuguDB/go-xugu-driver; fi
if [ "$race" = true ]; then go test -race ./...; else go test ./...; fi
if [ "$driver" = zookeeper ]; then
for target in darwin/arm64 darwin/amd64 linux/arm64 linux/amd64 windows/arm64 windows/amd64; do
IFS=/ read -r goos goarch <<< "$target"
output="/tmp/dbx-agent-zookeeper-${goos}-${goarch}"
[ "$goos" = windows ] && output="${output}.exe"
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags="-s -w" -o "$output" .
done
else
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o "/tmp/dbx-agent-${binary}-linux-x64" .
fi
.github/scripts/ci-agent-integration.sh
+192
View File
@@ -0,0 +1,192 @@
#!/usr/bin/env bash
set -euo pipefail
scenario="${1:?scenario is required}"
version="${2:?version is required}"
image="${3:-}"
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
case "$scenario" in
zookeeper)
cd "$root/agents/drivers/zookeeper"
name="dbx-zookeeper-${version//./-}"
docker rm -fv "$name" >/dev/null 2>&1 || true
docker run -d --name "$name" -p 2181:2181 "zookeeper:$version"
cleanup() {
docker rm -fv "$name" >/dev/null 2>&1 || true
}
trap cleanup EXIT
ready=false
for _ in $(seq 1 90); do
if timeout 1 bash -c '</dev/tcp/127.0.0.1/2181' >/dev/null 2>&1; then
ready=true
break
fi
sleep 2
done
if [ "$ready" != true ]; then
docker logs "$name"
exit 1
fi
DBX_ZOOKEEPER_TEST_CONNECT_STRING=127.0.0.1:2181 \
go test -run '^TestZooKeeperIntegration$' -count=1 ./...
if [ "$version" = "3.7.0" ]; then
DBX_ZOOKEEPER_TEST_CONNECT_STRING=127.0.0.1:2181 \
go test -run '^TestZooKeeperLargeChildrenIntegration$' -count=1 ./...
fi
;;
zookeeper-sasl)
cd "$root/agents/drivers/zookeeper"
name="dbx-zookeeper-sasl-3-7-0"
jaas_file=$(mktemp)
cat > "$jaas_file" <<'EOF'
Server {
org.apache.zookeeper.server.auth.DigestLoginModule required
user_dbx="dbx-secret";
};
EOF
chmod 0644 "$jaas_file"
docker rm -fv "$name" >/dev/null 2>&1 || true
docker run -d --name "$name" -p 2181:2181 \
-v "$jaas_file:/conf/server-jaas.conf:ro" \
-e ZOO_CFG_EXTRA="authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider" \
-e JVMFLAGS="-Djava.security.auth.login.config=/conf/server-jaas.conf -Dzookeeper.sasl.serverconfig=Server -Dzookeeper.sessionRequireClientSASLAuth=true" \
zookeeper:3.7.0
cleanup() {
docker rm -fv "$name" >/dev/null 2>&1 || true
}
trap cleanup EXIT
ready=false
for _ in $(seq 1 90); do
if timeout 1 bash -c '</dev/tcp/127.0.0.1/2181' >/dev/null 2>&1; then
ready=true
break
fi
sleep 2
done
if [ "$ready" != true ]; then
docker logs "$name"
exit 1
fi
DBX_ZOOKEEPER_TEST_CONNECT_STRING=127.0.0.1:2181 \
DBX_ZOOKEEPER_TEST_AUTH_SCHEME=sasl_digest \
DBX_ZOOKEEPER_TEST_USERNAME=dbx \
DBX_ZOOKEEPER_TEST_PASSWORD=dbx-secret \
go test -run '^TestZooKeeperIntegration$' -count=1 ./...
;;
rocketmq)
cd "$root/agents/drivers/rocketmq"
./scripts/run-integration.sh "$version"
;;
tdengine)
cd "$root/agents"
name="dbx-tdengine-${version//./-}"
docker rm -fv "$name" >/dev/null 2>&1 || true
docker run -d --name "$name" \
--memory 4g \
--ulimit nofile=65535:65535 \
-p 6030:6030 \
-p 6041:6041 \
"$image"
cleanup() {
docker rm -fv "$name" >/dev/null 2>&1 || true
}
trap cleanup EXIT
ready=false
dnodes=""
for _ in $(seq 1 90); do
dnodes="$(docker exec "$name" taos -s 'SHOW DNODES' 2>/dev/null || true)"
if grep -Eq '\|[[:space:]]*ready[[:space:]]*\|' <<<"$dnodes"; then
ready=true
break
fi
sleep 2
done
if [ "$ready" != "true" ]; then
printf '%s\n' "$dnodes"
docker logs "$name"
exit 1
fi
TDENGINE_INTEGRATION=1 \
TDENGINE_TEST_HOST=127.0.0.1 \
TDENGINE_TEST_PORT=6041 \
cargo test --manifest-path drivers/tdengine/Cargo.toml --locked --test live -- --nocapture
;;
cassandra)
cd "$root/agents/drivers/cassandra-go"
name="dbx-cassandra-${version//./-}"
docker rm -fv "$name" >/dev/null 2>&1 || true
docker run -d --name "$name" \
-e CASSANDRA_CLUSTER_NAME="DBX Cassandra CI $version" \
-e CASSANDRA_DC=dc1 \
-e CASSANDRA_RACK=rack1 \
-e CASSANDRA_ENDPOINT_SNITCH=GossipingPropertyFileSnitch \
-e CASSANDRA_NUM_TOKENS=16 \
-e MAX_HEAP_SIZE=512M \
-e HEAP_NEWSIZE=100M \
-p 9042:9042 \
"cassandra:$version"
cleanup() {
docker rm -fv "$name" >/dev/null 2>&1 || true
}
trap cleanup EXIT
ready=false
for _ in $(seq 1 100); do
if docker exec "$name" cqlsh -e 'SELECT release_version FROM system.local' >/dev/null 2>&1; then
ready=true
break
fi
sleep 3
done
if [ "$ready" != "true" ]; then
docker logs "$name"
exit 1
fi
CASSANDRA_TEST_HOST=127.0.0.1 \
CASSANDRA_TEST_PORT=9042 \
go test -run '^TestCassandraIntegration$' -count=1 ./...
;;
rabbitmq)
cd "$root/agents/drivers/rabbitmq"
name="dbx-rabbitmq-${version//./-}"
cookie="dbx-ci-${GITHUB_RUN_ID:-local}-${version//./-}"
# RabbitMQ is disposable in CI. Keep its data on a fresh tmpfs
# owned by the image's rabbitmq user so .erlang.cookie is readable.
docker rm -fv "$name" >/dev/null 2>&1 || true
docker run -d --name "$name" \
--user 999:999 \
--tmpfs /var/lib/rabbitmq:rw,exec,uid=999,gid=999,mode=700 \
-e RABBITMQ_DEFAULT_USER=dbx \
-e RABBITMQ_DEFAULT_PASS=dbx-password \
-e RABBITMQ_ERLANG_COOKIE="$cookie" \
-p 5672:5672 -p 15672:15672 \
"rabbitmq:${version}-management"
cleanup() {
docker rm -fv "$name" >/dev/null 2>&1 || true
}
trap cleanup EXIT
ready=false
for _ in $(seq 1 60); do
if docker exec "$name" rabbitmq-diagnostics -q check_running >/dev/null 2>&1 \
&& curl --fail --silent --noproxy '*' --user dbx:dbx-password \
http://127.0.0.1:15672/api/overview >/dev/null; then
ready=true
break
fi
sleep 2
done
if [ "$ready" != "true" ]; then
docker inspect "$name" --format 'image={{.Config.Image}} user={{.Config.User}} status={{.State.Status}} exit={{.State.ExitCode}}' || true
docker logs "$name"
exit 1
fi
RABBITMQ_INTEGRATION=1 \
RABBITMQ_USERNAME=dbx \
RABBITMQ_PASSWORD=dbx-password \
go test -run '^TestRabbitMQIntegration$' -count=1 ./...
;;
*)
echo "Unknown integration scenario: $scenario" >&2
exit 2
;;
esac
.github/scripts/ci-config.mjs
+32
View File
@@ -0,0 +1,32 @@
export const rustGroups = {
foundation: ["dbx-types", "dbx-platform", "dbx-sql", "dbx-formats", "dbx-ai-provider", "dbx-plugin-runtime"],
drivers: ["dbx-drivers", "dbx-sqlite-worker"],
application: ["dbx", "dbx-core", "dbx-web", "dbx-cli", "dbx-mcp"],
};
export const goAgents = [
{ driver: "oracle-go", binary: "oracle", race: false },
{ driver: "xugu", binary: "xugu", race: false },
{ driver: "rabbitmq", binary: "rabbitmq", race: false },
{ driver: "rocketmq", binary: "rocketmq", race: true },
{ driver: "zookeeper", binary: "zookeeper", race: true },
{ driver: "cassandra-go", binary: "cassandra", race: false },
{ driver: "hive-go", binary: "hive", race: false },
{ driver: "vastbase-go", binary: "vastbase", race: false },
{ driver: "neo4j-go", binary: "neo4j", race: false },
{ driver: "iotdb", binary: "iotdb", race: false },
];
export const rustAgents = ["duckdb", "tdengine"];
export const integrationCases = [
...["3.4.14", "3.5.5", "3.7.0", "3.9.5"].map((version) => ({ driver: "zookeeper", scenario: "zookeeper", version })),
{ driver: "zookeeper", scenario: "zookeeper-sasl", version: "3.7.0" },
...["4.9.8", "5.3.1"].map((version) => ({ driver: "rocketmq", scenario: "rocketmq", version })),
...["2.4.0.14", "2.6.0.34", "3.0.7.1", "3.3.6.13", "3.4.2.2"].map((version) => ({
driver: "tdengine", scenario: "tdengine", version,
image: `tdengine/${version === "3.4.2.2" ? "tsdb" : "tdengine"}:${version}`,
})),
...["3.11.19", "5.0.6"].map((version) => ({ driver: "cassandra-go", scenario: "cassandra", version })),
...["3.13", "4.3"].map((version) => ({ driver: "rabbitmq", scenario: "rabbitmq", version })),
];
.github/scripts/ci-execution.test.mjs
+121
View File
@@ -0,0 +1,121 @@
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { mkdtempSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import test from "node:test";
import { goAgents, integrationCases } from "./ci-config.mjs";
import { lockfileManifests } from "./ci-lockfiles.mjs";
const root = fileURLToPath(new URL("../../", import.meta.url));
const environmentKeys = ["CGO_ENABLED", "GOOS", "GOARCH", "GONOSUMDB", "DBX_ZOOKEEPER_TEST_CONNECT_STRING",
"DBX_ZOOKEEPER_TEST_AUTH_SCHEME", "DBX_ZOOKEEPER_TEST_USERNAME", "DBX_ZOOKEEPER_TEST_PASSWORD", "TDENGINE_INTEGRATION",
"TDENGINE_TEST_HOST", "TDENGINE_TEST_PORT", "CASSANDRA_TEST_HOST", "CASSANDRA_TEST_PORT", "RABBITMQ_INTEGRATION",
"RABBITMQ_USERNAME", "RABBITMQ_PASSWORD"];
function withToolStubs(run, extraEnv = {}) {
const directory = mkdtempSync(path.join(tmpdir(), "dbx-ci-execution-"));
const binaries = path.join(directory, "bin");
mkdirSync(binaries);
const log = path.join(directory, "commands.jsonl");
const stub = path.join(binaries, "stub.cjs");
writeFileSync(stub, `#!/usr/bin/env node
const fs = require('node:fs');
const path = require('node:path');
const tool = path.basename(process.argv[1]);
const args = process.argv.slice(2);
const env = Object.fromEntries(${JSON.stringify(environmentKeys)}.filter(name => process.env[name] !== undefined).map(name => [name, process.env[name]]));
fs.appendFileSync(process.env.CI_STUB_LOG, JSON.stringify({tool,args,cwd:process.cwd(),env})+'\\n');
if (process.env.CI_STUB_FAIL === tool) process.exit(23);
if (tool === 'docker' && args.includes('taos')) console.log('| ready |');
`, { mode: 0o755 });
for (const name of ["docker", "go", "cargo", "timeout", "curl", "sleep", "bash"]) symlinkSync(stub, path.join(binaries, name));
try {
const env = { ...process.env, ...extraEnv, TMPDIR: directory, CI_STUB_LOG: log, PATH: `${binaries}${path.delimiter}${process.env.PATH}` };
return run(env, () => readFileSync(log, "utf8").trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)));
} finally { rmSync(directory, { recursive: true, force: true }); }
}
for (const agent of goAgents) {
test(`Go job preserves ${agent.driver} tests and target builds without executing compilers`, () => withToolStubs((env, commands) => {
const result = spawnSync("/bin/bash", [".github/scripts/ci-agent-go.sh", agent.driver, agent.binary, String(agent.race)], { cwd: root, env, encoding: "utf8" });
assert.equal(result.status, 0, result.stderr);
const invocations = commands();
assert.deepEqual(invocations[0].args, agent.race ? ["test", "-race", "./..."] : ["test", "./..."]);
assert.ok(invocations.every((entry) => entry.cwd === path.join(root, "agents/drivers", agent.driver)));
const builds = invocations.slice(1);
assert.equal(builds.length, agent.driver === "zookeeper" ? 6 : 1);
for (const build of builds) {
assert.deepEqual(build.args.slice(0, 3), ["build", "-trimpath", "-ldflags=-s -w"]);
assert.equal(build.env.CGO_ENABLED, "0");
}
if (agent.driver === "zookeeper") assert.deepEqual(builds.map((entry) => `${entry.env.GOOS}/${entry.env.GOARCH}`),
["darwin/arm64", "darwin/amd64", "linux/arm64", "linux/amd64", "windows/arm64", "windows/amd64"]);
else assert.ok(builds[0].args.includes(`/tmp/dbx-agent-${agent.binary}-linux-x64`));
if (agent.driver === "xugu") assert.ok(invocations.every((entry) => entry.env.GONOSUMDB === "gitee.com/XuguDB/go-xugu-driver"));
}));
}
for (const entry of integrationCases) {
test(`live matrix retains ${entry.scenario}/${entry.version} commands without starting any container`, () => withToolStubs((env, commands) => {
const result = spawnSync("/bin/bash", [".github/scripts/ci-agent-integration.sh", entry.scenario, entry.version, entry.image ?? ""], { cwd: root, env, encoding: "utf8" });
assert.equal(result.status, 0, result.stderr);
const invocations = commands();
if (entry.scenario === "rocketmq") {
assert.ok(invocations.some((call) => call.tool === "bash" && call.args.at(-1) === entry.version));
return;
}
const start = invocations.find((call) => call.tool === "docker" && call.args[0] === "run");
assert.ok(start, "starts the selected container");
const expectedImage = entry.image ?? (entry.scenario.startsWith("zookeeper") ? `zookeeper:${entry.version}`
: entry.scenario === "rabbitmq" ? `rabbitmq:${entry.version}-management` : `cassandra:${entry.version}`);
assert.equal(start.args.at(-1), expectedImage);
const last = invocations.at(-1);
assert.equal(last.tool, "docker");
assert.deepEqual(last.args.slice(0, 2), ["rm", "-fv"]);
const tests = invocations.filter((call) => ["go", "cargo"].includes(call.tool));
assert.ok(tests.length);
if (entry.scenario === "zookeeper") {
assert.equal(tests.length, entry.version === "3.7.0" ? 2 : 1);
assert.ok(tests[0].args.includes("^TestZooKeeperIntegration$"));
if (entry.version === "3.7.0") assert.ok(tests[1].args.includes("^TestZooKeeperLargeChildrenIntegration$"));
} else if (entry.scenario === "zookeeper-sasl") {
assert.equal(tests[0].env.DBX_ZOOKEEPER_TEST_AUTH_SCHEME, "sasl_digest");
assert.equal(tests[0].env.DBX_ZOOKEEPER_TEST_USERNAME, "dbx");
assert.ok(start.args.some((arg) => arg.includes("sessionRequireClientSASLAuth=true")));
} else if (entry.scenario === "tdengine") {
assert.deepEqual(tests[0].args, ["test", "--manifest-path", "drivers/tdengine/Cargo.toml", "--locked", "--test", "live", "--", "--nocapture"]);
assert.equal(tests[0].env.TDENGINE_INTEGRATION, "1");
assert.equal(tests[0].env.TDENGINE_TEST_PORT, "6041");
} else if (entry.scenario === "cassandra") assert.ok(tests[0].args.includes("^TestCassandraIntegration$"));
else {
assert.ok(tests[0].args.includes("^TestRabbitMQIntegration$"));
assert.equal(tests[0].env.RABBITMQ_INTEGRATION, "1");
assert.ok(start.args.includes("999:999"));
assert.ok(start.args.includes("/var/lib/rabbitmq:rw,exec,uid=999,gid=999,mode=700"));
}
}));
}
test("failed live tests stay failed and still clean their container", () => withToolStubs((env, commands) => {
const result = spawnSync("/bin/bash", [".github/scripts/ci-agent-integration.sh", "cassandra", "5.0.6"], { cwd: root, env, encoding: "utf8" });
assert.equal(result.status, 23);
assert.deepEqual(commands().at(-1).args.slice(0, 2), ["rm", "-fv"]);
}, { CI_STUB_FAIL: "go" }));
test("lock preflight resolves full graphs, keeps --locked and propagates failures", () => {
const plan = { rust: true, agent_rust: { include: [{ driver: "duckdb" }, { driver: "tdengine" }] } };
assert.deepEqual(lockfileManifests(plan), ["Cargo.toml", "agents/drivers/duckdb/Cargo.toml", "agents/drivers/tdengine/Cargo.toml"]);
assert.throws(() => lockfileManifests({ rust: false, agent_rust: { include: [{ driver: "../../unknown" }] } }));
for (const failure of ["", "cargo"]) withToolStubs((env, commands) => {
const result = spawnSync(process.execPath, [".github/scripts/ci-lockfiles.mjs"], { cwd: root, env, encoding: "utf8" });
assert.equal(result.status, failure ? 23 : 0, result.stderr);
assert.equal(commands().length, failure ? 1 : 3);
for (const command of commands()) {
assert.equal(command.args[0], "metadata");
assert.ok(command.args.includes("--locked"));
assert.ok(!command.args.includes("--no-deps"));
}
}, { CI_PLAN: JSON.stringify(plan), CI_STUB_FAIL: failure });
});
.github/scripts/ci-gate.mjs
+40
View File
@@ -0,0 +1,40 @@
import path from "node:path";
import { pathToFileURL } from "node:url";
export function gateFailures(needs, mode) {
if (needs.changes?.result !== "success") return ["changes did not succeed"];
let plan;
try { plan = JSON.parse(needs.changes.outputs.plan); } catch { return ["missing or invalid CI plan"]; }
if (!plan || typeof plan !== "object" || Array.isArray(plan)) return ["incomplete CI plan"];
const flags = ["rust", "agents", "fast", "agent_java", "agent_go_changed", "agent_rust_changed", "agent_integration_changed"];
if (!flags.every((flag) => typeof plan[flag] === "boolean")) return ["incomplete CI plan"];
const routedJobs = { frontend: "frontend", packages: "packages", "github-scripts": "github_scripts",
"windows-win7-bundle": "windows_win7_bundle", "duckdb-windows-driver": "duckdb_windows",
jdbc: "jdbc", "offline-jdbc-release": "offline_jdbc", "nix-packaging": "nix" };
if (mode === "all" && !Object.values(routedJobs).every((output) => ["true", "false"].includes(needs.changes.outputs[output]))) {
return ["missing or invalid job selection outputs"];
}
const expected = mode === "rust" ? {
"fast-checks": plan.fast, "rust-fmt-clippy": plan.rust, "rust-test": plan.rust,
} : mode === "agents" ? {
"fast-checks": plan.fast, "agent-checks": plan.agents, "agent-java": plan.agent_java, "agent-go": plan.agent_go_changed,
"agent-rust": plan.agent_rust_changed, "agent-integration": plan.agent_integration_changed,
} : mode === "all" ? {
rust: true, agents: true, "fast-checks": plan.fast,
...Object.fromEntries(Object.entries(routedJobs)
.map(([job, output]) => [job, needs.changes.outputs[output] === "true"])),
} : null;
if (!expected) return [`unknown gate: ${mode}`];
return Object.entries(expected).flatMap(([job, required]) => {
const result = needs[job]?.result;
return result === "success" || (!required && result === "skipped") ? [] : [`${job}: ${result ?? "missing"} (required=${required})`];
});
}
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
const failures = gateFailures(JSON.parse(process.env.NEEDS_JSON), process.argv[2]);
if (failures.length) {
console.error(failures.join("\n"));
process.exitCode = 1;
} else console.log("All selected CI jobs succeeded; only unselected jobs may be skipped.");
}
.github/scripts/ci-guide.md
+108
View File
@@ -0,0 +1,108 @@
# CI job boundaries
Paths in this guide are relative to `.github/`.
`workflows/ci.yml` retains one workflow and one cancellation group. Splitting jobs
does not change release workflows or branch-protection settings. The existing
`rust` and `agents` aggregate check names remain; `ci` additionally summarizes all
selected jobs. None of these gates accepts a failed, cancelled, missing, or
unexpectedly skipped prerequisite.
## Selection
`scripts/ci-plan.mjs` reads Cargo workspace metadata and the exact event base/head
diff. Renames include both paths and deleted paths remain visible. Rust package
selection follows reverse dependencies, including build and test dependencies.
Connection-type descriptors belong to `dbx-types`; dialect descriptors belong to
`dbx-sql`. A changed core dependency also selects the standalone DuckDB Agent tests,
whose development dependency is `dbx-core`.
- Ordinary Rust PRs select foundation, driver, and application test groups from
`scripts/ci-config.mjs`. Each group contains several packages, not one runner per
crate. At most three Rust test groups run concurrently.
- Main pushes, shared Cargo manifests/locks, toolchain/config/vendor changes, CI
infrastructure changes, an unknown workspace member, or an unavailable diff
select the original full-workspace test lane instead of duplicating it with all
three groups.
- Strict clippy still checks the complete workspace and all targets. The existing
full/fast feature distinction is retained: fast skips system-font discovery,
not the other previously enabled capabilities.
- Native Agent source changes select the affected native driver and its live
cases. Shared Agent inputs conservatively select all existing Agent checks.
Windows DuckDB packaging is selected by its own inputs or shared build/lock
changes, not every change to its development dependency's source.
- Existing frontend, package, Windows, JDBC, offline-payload and Nix jobs retain
their commands and path filters.
Changes to the planner itself select the broad checks. Missing/invalid planning
data fails the aggregate gates rather than producing a successful skip.
## Fast checks and Rust feature coverage
`fast-checks` runs formatting and CI/architecture contracts before dependency
resolution or compilation. It checks generated connection types without rewriting
them and validates the relevant root and independent Agent Cargo lockfiles with
full `cargo metadata --locked`; `--no-deps` is intentionally not used for that
validation.
Cargo may unify features supplied by different workspace consumers. Grouped tests
therefore explicitly retain the features previously enabled on each package,
including plugin runtime defaults and the driver's bundled SQLite capability.
`scripts/ci-rust-coverage.mjs` compares each selected workspace package's enabled
features with the full-workspace invocation and verifies that the groups cover
every workspace member exactly once. It fails if a selected package loses a
feature. An unassigned new workspace member forces the full-workspace lane and
bypasses only the group-specific audit until it is assigned a group. This does not substitute for
the full-workspace integration lane, and does not use `--all-features` to combine
incompatible SQLite backends.
## Agent jobs
| Job | Coverage | Maximum parallelism |
| --- | --- | --- |
| `agent-checks` | Existing Python/script tests and Agent source validation | 1 |
| `agent-java` | Existing Java tests, `shadowJar`, and JAR validation | 1 |
| `agent-rust` | DuckDB and TDengine tests; TDengine release build | 2 |
| `agent-go` | Ten existing native Go test/build targets, original race flags, six ZooKeeper cross-build targets | 4 |
| `agent-integration` | Sixteen existing engine/version/authentication cases | 4 |
Matrix jobs set `fail-fast: false`, so one failure does not conceal results from
the other selected cases. Live test predicates, readiness checks, time limits,
credentials for disposable fixtures, and failure cleanup are retained.
Native, Java and live jobs wait for fast checks and Agent source validation before
starting expensive builds or containers.
RocketMQ's existing integration harness starts official server JARs through Maven
and Java rather than Docker; those matrix entries explicitly retain Java 21.
| Integration scenario | Versions |
| --- | --- |
| ZooKeeper | 3.4.14, 3.5.5, 3.7.0, 3.9.5; large-child-list coverage remains on 3.7.0 |
| ZooKeeper SASL DIGEST-MD5 | 3.7.0 |
| RocketMQ | 4.9.8, 5.3.1 |
| TDengine | 2.4.0.14, 2.6.0.34, 3.0.7.1, 3.3.6.13, 3.4.2.2; the last uses `tdengine/tsdb` |
| Cassandra | 3.11.19, 5.0.6 |
| RabbitMQ | 3.13, 4.3 |
## Validation
Run these from the repository root:
```sh
node --test .github/scripts/ci-*.test.mjs
node .github/scripts/ci-rust-coverage.mjs
actionlint .github/workflows/ci.yml
bash -n .github/scripts/ci-agent-go.sh
bash -n .github/scripts/ci-agent-integration.sh
```
The shell-command tests replace Docker, compilers, readiness probes and delegated
integration scripts with recording stubs. They verify all configured cases and
failure propagation without starting an engine; they are not live database tests.
Do not run the integration shell script on a developer machine. It targets
disposable Actions runners; equivalent validation on the designated test server
must follow that server's isolated-container naming and port policies.
The split aims to isolate failures, return small-check results earlier, and remove
unrelated work. Rust groups can still repeat dependency compilation across runners;
compare cold/warm build, link, test and queue time on actual runs before claiming a
speedup or changing group sizes. Cancelled runs are not successful benchmarks.
.github/scripts/ci-lockfiles.mjs
+23
View File
@@ -0,0 +1,23 @@
import { spawnSync } from "node:child_process";
import path from "node:path";
import { pathToFileURL } from "node:url";
export function lockfileManifests(plan) {
return [...(plan.rust ? ["Cargo.toml"] : []),
...plan.agent_rust.include.map(({ driver }) => {
if (!["duckdb", "tdengine"].includes(driver)) throw new Error(`Unknown native Rust driver: ${driver}`);
return `agents/drivers/${driver}/Cargo.toml`;
})];
}
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
const plan = JSON.parse(process.env.CI_PLAN);
for (const manifest of lockfileManifests(plan)) {
console.log(`Checking locked dependency resolution: ${manifest}`);
const result = spawnSync("cargo", ["metadata", "--locked", "--manifest-path", manifest, "--format-version", "1"], {
stdio: ["ignore", "ignore", "inherit"],
});
if (result.error) throw result.error;
if (result.status !== 0) process.exit(result.status ?? 1);
}
}
.github/scripts/ci-plan.mjs
+93
View File
@@ -0,0 +1,93 @@
import { appendFileSync } from "node:fs";
import { execFileSync } from "node:child_process";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { goAgents, integrationCases, rustAgents, rustGroups } from "./ci-config.mjs";
export function changedPaths(base, root) {
if (!base || /^0+$/.test(base)) return null;
execFileSync("git", ["rev-parse", "--verify", `${base}^{commit}`], { cwd: root, stdio: "pipe" });
return execFileSync("git", ["diff", "--no-renames", "--name-only", "-z", base, "HEAD", "--"], {
cwd: root, encoding: "utf8", maxBuffer: 16 * 1024 * 1024,
}).split("\0").filter(Boolean);
}
export function planCi({ files, metadata, root, eventName = "pull_request", rustChanged = false, agentsChanged = false }) {
const unknownDiff = files === null;
files ??= [];
const packages = metadata.packages.filter((pkg) => metadata.workspace_members.includes(pkg.id));
const members = new Map(packages.map((pkg) => [pkg.name, pkg]));
const packagePaths = packages.map((pkg) => [pkg.name, `${path.relative(root, path.dirname(pkg.manifest_path)).split(path.sep).join("/")}/`]);
const ciChanged = unknownDiff || files.some((file) => /^\.github\/(?:workflows\/ci(?:[.-]|\/)|scripts\/ci-|actions\/ci-)/.test(file));
const sharedRust = ciChanged || files.some((file) => /^(?:Cargo\.(?:toml|lock)$|rust-toolchain|\.cargo\/|vendor\/)/.test(file)
|| /^(?:crates\/[^/]+|src-tauri)\/Cargo\.toml$/.test(file));
const affected = new Set();
let unknownRust = false;
for (const file of files) {
const owner = packagePaths.find(([, directory]) => file.startsWith(directory));
if (owner) affected.add(owner[0]);
else if (/^(?:crates|src-tauri)\//.test(file)) unknownRust = true;
if (file.startsWith("plugins/connection-types/")) affected.add("dbx-types");
if (file.startsWith("plugins/dialects/")) affected.add("dbx-sql");
}
const knownGroups = new Set(Object.values(rustGroups).flat());
const unknownMember = packages.some((pkg) => !knownGroups.has(pkg.name));
const rust = rustChanged || sharedRust || unknownRust || affected.size > 0 || files.includes("scripts/core-architecture.test.mjs");
const full = rust && (eventName !== "pull_request" || sharedRust || unknownRust || unknownMember || affected.size === 0);
if (full) for (const pkg of packages) affected.add(pkg.name);
let expanded;
do {
expanded = false;
for (const pkg of packages) {
if (!affected.has(pkg.name) && pkg.dependencies.some((dependency) => members.has(dependency.name) && affected.has(dependency.name))) {
affected.add(pkg.name);
expanded = true;
}
}
} while (expanded);
const rustMatrix = !rust ? [] : full ? [{ group: "workspace" }] : Object.entries(rustGroups)
.filter(([, names]) => names.some((name) => affected.has(name)))
.map(([group]) => ({ group }));
const nativeDrivers = [...goAgents.map((agent) => agent.driver), ...rustAgents];
const nativeChanges = new Set(nativeDrivers.filter((driver) => files.some((file) => file.startsWith(`agents/drivers/${driver}/`))));
const sharedAgents = ciChanged || sharedRust || files.some((file) => file.startsWith("agents/")
&& !nativeDrivers.some((driver) => file.startsWith(`agents/drivers/${driver}/`)))
|| files.some((file) => file.startsWith(".github/scripts/bump-agent-versions.") || file === ".github/workflows/agents-release.yml"
|| file === "crates/dbx-drivers/assets/agent-protocol-v2.json");
const allAgents = sharedAgents || (agentsChanged && nativeChanges.size === 0);
if (allAgents) {
for (const driver of nativeDrivers) nativeChanges.add(driver);
}
if (affected.has("dbx-core")) nativeChanges.add("duckdb");
const goMatrix = goAgents.filter((agent) => nativeChanges.has(agent.driver));
const nativeRustMatrix = rustAgents.filter((driver) => nativeChanges.has(driver)).map((driver) => ({ driver }));
const liveMatrix = integrationCases.filter((entry) => nativeChanges.has(entry.driver));
const java = allAgents;
const agents = agentsChanged || sharedAgents || nativeChanges.size > 0;
return {
rust, rust_full: full, rust_matrix: { include: rustMatrix }, rust_groups_known: !unknownMember,
affected_packages: [...affected].sort(),
agents, agent_java: java, agent_go: { include: goMatrix }, agent_rust: { include: nativeRustMatrix },
agent_integration: { include: liveMatrix }, agent_go_changed: goMatrix.length > 0,
agent_rust_changed: nativeRustMatrix.length > 0, agent_integration_changed: liveMatrix.length > 0,
duckdb_changed: nativeChanges.has("duckdb"),
duckdb_windows: sharedRust || files.some((file) => file.startsWith("agents/drivers/duckdb/")),
fast: rust || agents || ciChanged,
};
}
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
const root = process.cwd();
const metadata = JSON.parse(execFileSync("cargo", ["metadata", "--locked", "--offline", "--no-deps", "--format-version", "1"], {
cwd: root, encoding: "utf8", maxBuffer: 16 * 1024 * 1024,
}));
const plan = planCi({
files: changedPaths(process.env.BASE_SHA, root), metadata, root, eventName: process.env.GITHUB_EVENT_NAME,
rustChanged: process.env.RUST_CHANGED === "true", agentsChanged: process.env.AGENTS_CHANGED === "true",
});
if (process.env.GITHUB_OUTPUT) {
const outputs = { ...plan, plan };
appendFileSync(process.env.GITHUB_OUTPUT, Object.entries(outputs).map(([name, value]) => `${name}=${JSON.stringify(value)}\n`).join(""));
}
console.log(JSON.stringify(plan, null, 2));
}
.github/scripts/ci-plan.test.mjs
+232
View File
@@ -0,0 +1,232 @@
import assert from "node:assert/strict";
import test from "node:test";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { execFileSync } from "node:child_process";
import { goAgents, integrationCases, rustGroups } from "./ci-config.mjs";
import { changedPaths, planCi } from "./ci-plan.mjs";
import { rustCommand } from "./ci-rust.mjs";
import { gateFailures } from "./ci-gate.mjs";
import { assertCoverage, parseCoverage } from "./ci-rust-coverage.mjs";
const root = "/fixture";
const graph = {
"dbx-types": [], "dbx-platform": [], "dbx-formats": [], "dbx-sql": ["dbx-types"],
"dbx-ai-provider": ["dbx-platform"], "dbx-plugin-runtime": ["dbx-platform", "dbx-types"],
"dbx-sqlite-worker": [], "dbx-drivers": ["dbx-platform", "dbx-sql", "dbx-types", "dbx-sqlite-worker"],
"dbx-core": ["dbx-drivers", "dbx-sql", "dbx-types", "dbx-platform", "dbx-ai-provider", "dbx-plugin-runtime", "dbx-formats"],
"dbx-mcp": ["dbx-core"], "dbx-cli": ["dbx-core", "dbx-mcp"], "dbx-web": ["dbx-core", "dbx-mcp"], "dbx": ["dbx-core", "dbx-mcp"],
};
const metadata = {
workspace_members: Object.keys(graph),
packages: Object.entries(graph).map(([name, dependencies]) => ({
id: name, name, manifest_path: `${root}/${name === "dbx" ? "src-tauri" : `crates/${name}`}/Cargo.toml`,
dependencies: dependencies.map((dependency) => ({ name: dependency })),
})),
};
const plan = (files, options = {}) => planCi({ files, metadata, root, ...options });
const groups = (result) => result.rust_matrix.include.map((entry) => entry.group);
test("foundation changes select transitive consumers and standalone DuckDB", () => {
const result = plan(["crates/dbx-types/src/lib.rs"]);
assert.deepEqual(groups(result), ["foundation", "drivers", "application"]);
for (const name of ["dbx-types", "dbx-sql", "dbx-drivers", "dbx-core", "dbx-web", "dbx-cli", "dbx-mcp", "dbx"]) {
assert.ok(result.affected_packages.includes(name), name);
}
assert.deepEqual(result.agent_rust.include, [{ driver: "duckdb" }]);
assert.equal(result.duckdb_changed, true);
assert.equal(result.duckdb_windows, false);
assert.equal(result.rust_groups_known, true);
assert.equal(result.agent_java, false);
});
test("core and consumer changes do not pull unrelated foundation test groups", () => {
assert.deepEqual(groups(plan(["crates/dbx-core/src/query/mod.rs"])), ["application"]);
const cli = plan(["crates/dbx-cli/src/main.rs"]);
assert.deepEqual(groups(cli), ["application"]);
assert.equal(cli.agents, false);
assert.deepEqual(groups(plan(["crates/dbx-drivers/src/db/postgres.rs"])), ["drivers", "application"]);
});
test("code generation and test-only dependencies participate in impact analysis", () => {
assert.deepEqual(groups(plan(["plugins/connection-types/postgres.yaml"])), ["foundation", "drivers", "application"]);
assert.deepEqual(groups(plan(["plugins/dialects/postgres.yaml"])), ["foundation", "drivers", "application"]);
const fixture = structuredClone(metadata);
fixture.packages.find((pkg) => pkg.name === "dbx-types").dependencies.push({ name: "dbx-formats", kind: "dev" });
assert.ok(plan(["crates/dbx-formats/src/lib.rs"], { metadata: fixture }).affected_packages.includes("dbx-types"));
});
for (const file of ["Cargo.toml", "Cargo.lock", ".cargo/config.toml", "rust-toolchain.toml", "vendor/wry/src/lib.rs",
"crates/dbx-sql/Cargo.toml", ".github/workflows/ci.yml", ".github/scripts/ci-plan.mjs", ".github/scripts/ci-gate.test.mjs"]) {
test(`shared input ${file} retains full workspace and Agent coverage`, () => {
const result = plan([file]);
assert.deepEqual(groups(result), ["workspace"]);
assert.equal(result.rust_full, true);
assert.equal(result.agent_go.include.length, 10);
assert.equal(result.agent_rust.include.length, 2);
assert.equal(result.agent_integration.include.length, 16);
assert.equal(result.agent_java, true);
assert.equal(result.duckdb_windows, true);
});
}
test("main and unknown graph changes fail open to full test coverage", () => {
assert.deepEqual(groups(plan(["crates/dbx-core/src/lib.rs"], { eventName: "push" })), ["workspace"]);
assert.deepEqual(groups(plan(["crates/new-engine/src/lib.rs"])), ["workspace"]);
assert.deepEqual(groups(plan(null)), ["workspace"]);
const fixture = structuredClone(metadata);
fixture.packages.push({ name: "new-member", id: "new-member", manifest_path: `${root}/crates/new-member/Cargo.toml`, dependencies: [] });
fixture.workspace_members.push("new-member");
assert.deepEqual(groups(plan(["crates/dbx-core/src/lib.rs"], { metadata: fixture })), ["workspace"]);
assert.equal(plan(["crates/dbx-core/src/lib.rs"], { metadata: fixture }).rust_groups_known, false);
assert.deepEqual(groups(plan(["scripts/core-architecture.test.mjs"])), ["workspace"]);
});
test("unrelated docs skip Rust and Agents without inventing a matrix row", () => {
const result = plan(["docs/README.md"]);
assert.equal(result.rust, false);
assert.equal(result.agents, false);
assert.equal(result.fast, false);
assert.deepEqual(result.rust_matrix, { include: [] });
});
for (const driver of ["duckdb", "tdengine", ...goAgents.map((entry) => entry.driver)]) {
test(`native ${driver} changes only select their Agent tests and live cases`, () => {
const result = plan([`agents/drivers/${driver}/source`], { agentsChanged: true });
assert.equal(result.rust, false);
assert.equal(result.agent_java, false);
assert.ok([...result.agent_go.include, ...result.agent_rust.include].every((entry) => entry.driver === driver));
assert.ok(result.agent_integration.include.every((entry) => entry.driver === driver));
});
}
test("shared Agent inputs and unknown native modules never silently lose coverage", () => {
for (const file of ["agents/common/src/main/java/Protocol.java", "agents/scripts/validate_agents.py", "agents/build.gradle",
"agents/drivers/new-driver/main.go", "crates/dbx-drivers/assets/agent-protocol-v2.json", ".github/workflows/agents-release.yml"]) {
const result = plan([file]);
assert.equal(result.agent_go.include.length, 10, file);
assert.equal(result.agent_integration.include.length, 16, file);
assert.equal(result.agent_java, true, file);
}
const fallback = plan(["future-agent-filter-input"], { agentsChanged: true });
assert.equal(fallback.agent_go.include.length, 10);
assert.equal(fallback.agent_integration.include.length, 16);
assert.equal(fallback.agent_java, true);
});
test("the independent live matrix preserves every old version and authentication case", () => {
assert.deepEqual(integrationCases.map((entry) => `${entry.scenario}:${entry.version}`), [
"zookeeper:3.4.14", "zookeeper:3.5.5", "zookeeper:3.7.0", "zookeeper:3.9.5", "zookeeper-sasl:3.7.0",
"rocketmq:4.9.8", "rocketmq:5.3.1", "tdengine:2.4.0.14", "tdengine:2.6.0.34", "tdengine:3.0.7.1",
"tdengine:3.3.6.13", "tdengine:3.4.2.2", "cassandra:3.11.19", "cassandra:5.0.6", "rabbitmq:3.13", "rabbitmq:4.3",
]);
assert.equal(integrationCases.find((entry) => entry.version === "3.4.2.2").image, "tdengine/tsdb:3.4.2.2");
assert.deepEqual(goAgents.filter((entry) => entry.race).map((entry) => entry.driver), ["rocketmq", "zookeeper"]);
});
test("test groups partition every current workspace package exactly once", () => {
const selected = Object.values(rustGroups).flat();
assert.equal(new Set(selected).size, selected.length);
assert.deepEqual(selected.toSorted(), Object.keys(graph).toSorted());
});
test("workspace Cargo flags preserve full versus fast coverage and strict clippy", () => {
for (const mode of ["fast", "full"]) {
const command = rustCommand("test", "workspace", mode);
const features = command.at(-1).split(",");
assert.ok(command.includes("--workspace"));
assert.ok(command.includes("--locked"));
for (const pkg of ["dbx", "dbx-core", "dbx-web"]) {
for (const feature of ["duckdb-sidecar", "dynamodb", "mq-admin", "sqlite-sqlcipher"]) assert.ok(features.includes(`${pkg}/${feature}`));
assert.equal(features.includes(`${pkg}/system-fonts`), mode === "full");
}
assert.deepEqual(rustCommand("clippy", "workspace", mode).slice(-3), ["--", "-D", "warnings"]);
}
for (const group of Object.keys(rustGroups)) assert.ok(rustCommand("test", group, "fast").includes("--package"));
assert.throws(() => rustCommand("test", "unknown", "full"));
assert.throws(() => rustCommand("clippy", "foundation", "full"));
});
function results(files) {
const selected = plan(files);
return { changes: { result: "success", outputs: { plan: JSON.stringify(selected) } },
"fast-checks": { result: selected.fast ? "success" : "skipped" },
"rust-fmt-clippy": { result: selected.rust ? "success" : "skipped" }, "rust-test": { result: selected.rust ? "success" : "skipped" },
"agent-checks": { result: selected.agents ? "success" : "skipped" }, "agent-java": { result: selected.agent_java ? "success" : "skipped" },
"agent-go": { result: selected.agent_go_changed ? "success" : "skipped" },
"agent-rust": { result: selected.agent_rust_changed ? "success" : "skipped" },
"agent-integration": { result: selected.agent_integration_changed ? "success" : "skipped" } };
}
test("gates accept only successful selected jobs and intentionally skipped unselected jobs", () => {
for (const files of [["docs/README.md"], ["Cargo.lock"], ["agents/drivers/rabbitmq/main.go"]]) {
for (const mode of ["rust", "agents"]) assert.deepEqual(gateFailures(results(files), mode), []);
}
for (const result of ["skipped", "failure", "cancelled", undefined]) {
const needs = results(["Cargo.lock"]);
needs["rust-test"].result = result;
assert.ok(gateFailures(needs, "rust").length);
needs["agent-integration"].result = result;
assert.ok(gateFailures(needs, "agents").length);
}
const needs = results(["docs/README.md"]);
needs["agent-go"].result = "cancelled";
assert.ok(gateFailures(needs, "agents").length);
needs.changes.result = "failure";
assert.ok(gateFailures(needs, "rust").length);
assert.ok(gateFailures({ changes: { result: "success", outputs: { plan: "{}" } } }, "rust").length);
assert.ok(gateFailures({ changes: { result: "success", outputs: { plan: "null" } } }, "rust").length);
});
test("git diff routing includes both sides of renames, deleted files, and unusual filenames", () => {
const directory = mkdtempSync(path.join(tmpdir(), "dbx-ci-plan-"));
const git = (...args) => execFileSync("git", args, { cwd: directory, encoding: "utf8", stdio: "pipe" }).trim();
try {
git("init"); git("config", "user.email", "fixture@example.invalid"); git("config", "user.name", "CI Fixture");
writeFileSync(path.join(directory, "old.rs"), "unchanged source\n");
writeFileSync(path.join(directory, "deleted.rs"), "gone\n");
git("add", "."); git("-c", "core.hooksPath=/dev/null", "commit", "-m", "fixture baseline");
const base = git("rev-parse", "HEAD");
git("mv", "old.rs", "renamed\nfile.rs"); git("rm", "deleted.rs");
git("-c", "core.hooksPath=/dev/null", "commit", "-m", "fixture change");
assert.deepEqual(changedPaths(base, directory).toSorted(), ["old.rs", "deleted.rs", "renamed\nfile.rs"].toSorted());
assert.equal(changedPaths("0".repeat(40), directory), null);
assert.throws(() => changedPaths("not-a-ref", directory));
} finally { rmSync(directory, { recursive: true, force: true }); }
});
test("the final gate rejects absent routing outputs and skipped selected jobs", () => {
const needs = results(["docs/README.md"]);
const routed = { frontend: "frontend", packages: "packages", "github-scripts": "github_scripts",
"windows-win7-bundle": "windows_win7_bundle", "duckdb-windows-driver": "duckdb_windows", jdbc: "jdbc",
"offline-jdbc-release": "offline_jdbc", "nix-packaging": "nix" };
needs.rust = needs.agents = { result: "success" };
for (const [job, output] of Object.entries(routed)) {
needs[job] = { result: "skipped" };
needs.changes.outputs[output] = "false";
}
assert.deepEqual(gateFailures(needs, "all"), []);
needs.changes.outputs.frontend = "true";
assert.ok(gateFailures(needs, "all").length);
needs.frontend.result = "success";
assert.deepEqual(gateFailures(needs, "all"), []);
delete needs.changes.outputs.nix;
assert.ok(gateFailures(needs, "all").length);
});
test("coverage audit rejects lost features, omitted packages, and overlapping group selection", () => {
const coverage = { workspace: Object.fromEntries(Object.keys(graph).map((name) => [name, ["fixture"]])),
...Object.fromEntries(Object.entries(rustGroups).map(([group, packages]) => [group, Object.fromEntries(packages.map((name) => [name, ["fixture"]]))])) };
assertCoverage(coverage);
const lost = structuredClone(coverage);
lost.foundation["dbx-types"] = [];
assert.throws(() => assertCoverage(lost), /lost workspace features/);
const missing = structuredClone(coverage);
delete missing.drivers["dbx-drivers"];
assert.throws(() => assertCoverage(missing), /wrong packages/);
const overlap = structuredClone(coverage);
overlap.foundation["dbx-core"] = ["fixture"];
assert.throws(() => assertCoverage(overlap), /wrong packages/);
assert.deepEqual(parseCoverage("dbx-core v0.1.0 (/fixture)|default,openapi\n"), { "dbx-core": ["default", "openapi"] });
});
.github/scripts/ci-rust-coverage.mjs
+39
View File
@@ -0,0 +1,39 @@
import assert from "node:assert/strict";
import { execFileSync } from "node:child_process";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { rustGroups } from "./ci-config.mjs";
import { rustCommand } from "./ci-rust.mjs";
export function parseCoverage(output) {
return Object.fromEntries(output.trim().split("\n").filter(Boolean).map((line) => {
const [name, features] = line.split("|");
assert.ok(features !== undefined, `Invalid Cargo tree output: ${line}`);
return [name.split(" ")[0], features.split(",").filter(Boolean)];
}));
}
export function assertCoverage(coverage) {
const names = Object.values(rustGroups).flat();
assert.equal(new Set(names).size, names.length, "Rust test groups overlap");
assert.deepEqual(names.toSorted(), Object.keys(coverage.workspace).toSorted(), "Rust test groups must cover every workspace package");
for (const [group, packages] of Object.entries(rustGroups)) {
assert.deepEqual(Object.keys(coverage[group]).toSorted(), packages.toSorted(), `${group} selected the wrong packages`);
for (const name of packages) {
const missing = coverage.workspace[name].filter((feature) => !coverage[group][name].includes(feature));
assert.deepEqual(missing, [], `${group}/${name} lost workspace features`);
}
}
}
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
const coverage = {};
for (const group of ["workspace", ...Object.keys(rustGroups)]) {
const args = rustCommand("test", group, "fast");
args[0] = "tree";
args.push("--depth", "0", "--prefix", "none", "--format", "{p}|{f}");
coverage[group] = parseCoverage(execFileSync("cargo", args, { encoding: "utf8", maxBuffer: 8 * 1024 * 1024, stdio: ["ignore", "pipe", "inherit"] }));
}
assertCoverage(coverage);
console.log(JSON.stringify(coverage, null, 2));
}
.github/scripts/ci-rust.mjs
+31
View File
@@ -0,0 +1,31 @@
import { spawnSync } from "node:child_process";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { rustGroups } from "./ci-config.mjs";
export function rustCommand(action, group, mode) {
if (!["test", "clippy"].includes(action) || !["full", "fast"].includes(mode)
|| (group !== "workspace" && !Object.hasOwn(rustGroups, group)) || (action === "clippy" && group !== "workspace")) {
throw new Error(`Unknown Rust CI configuration: ${action}/${group}/${mode}`);
}
const capabilityFeatures = ["duckdb-sidecar", "dynamodb", "mq-admin", "sqlite-sqlcipher"];
if (mode === "full") capabilityFeatures.push("system-fonts");
const appFeatures = ["dbx", "dbx-core", "dbx-web"].flatMap((name) => capabilityFeatures.map((feature) => `${name}/${feature}`));
const foundationFeatures = ["dbx-types/mq-admin", "dbx-types/openapi", "dbx-sql/duckdb-sidecar", "dbx-sql/openapi",
"dbx-platform/downloads", "dbx-platform/host-prompts", "dbx-platform/test-support", "dbx-plugin-runtime/default", "dbx-plugin-runtime/test-support"];
const features = group === "foundation" ? foundationFeatures : group === "drivers" ? [
...["duckdb-sidecar", "dynamodb", "mq-admin", "sqlite-bundled", "sqlite-sqlcipher", "test-support"].map((feature) => `dbx-drivers/${feature}`),
"dbx-sqlite-worker/runtime", "dbx-types/openapi", "dbx-sql/openapi",
] : appFeatures;
const packages = group === "workspace" ? ["--workspace"] : rustGroups[group].flatMap((name) => ["--package", name]);
return [action, ...packages, "--locked", ...(action === "clippy" ? ["--all-targets"] : []),
"--no-default-features", "--features", features.join(","), ...(action === "clippy" ? ["--", "-D", "warnings"] : [])];
}
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
const args = rustCommand(...process.argv.slice(2, 5));
console.log(`cargo ${args.join(" ")}`);
const result = spawnSync("cargo", args, { stdio: "inherit" });
if (result.error) throw result.error;
process.exit(result.status ?? 1);
}
.github/scripts/ci-workflow.test.mjs
+73
View File
@@ -0,0 +1,73 @@
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import test from "node:test";
const workflow = readFileSync(new URL("../workflows/ci.yml", import.meta.url), "utf8");
function job(name) {
const jobs = workflow.slice(workflow.indexOf("\njobs:\n") + 7);
const pattern = /^ ([\w-]+):\s*$/gm;
const definitions = [...jobs.matchAll(pattern)];
const index = definitions.findIndex((match) => match[1] === name);
assert.ok(index >= 0, `missing job ${name}`);
return jobs.slice(definitions[index].index, definitions[index + 1]?.index ?? jobs.length);
}
test("fast checks run format and contracts before graph resolution or compilation", () => {
const fast = job("fast-checks");
for (const command of ["cargo fmt --check", "node --test scripts/core-architecture.test.mjs",
"node scripts/sync-connection-types.mjs --check", "node .github/scripts/ci-lockfiles.mjs", "node .github/scripts/ci-rust-coverage.mjs"]) assert.ok(fast.includes(command));
assert.ok(fast.indexOf("cargo fmt --check") < fast.indexOf("ci-lockfiles.mjs"));
assert.doesNotMatch(fast, /cargo (?:test|build|clippy)/);
assert.ok(fast.includes("needs.changes.outputs.rust_groups_known == 'true'"));
});
test("Agent and Rust matrices are bounded and do not cancel sibling failures", () => {
for (const [name, output, parallel] of [["rust-test", "rust_matrix", 3], ["agent-rust", "agent_rust", 2],
["agent-go", "agent_go", 4], ["agent-integration", "agent_integration", 4]]) {
const content = job(name);
assert.match(content, /fail-fast: false/);
assert.ok(content.includes(`max-parallel: ${parallel}`));
assert.ok(content.includes(`fromJSON(needs.changes.outputs.${output})`));
assert.match(content, /if: needs\.changes\.outputs\.[\w_]+ == 'true'/);
if (name.startsWith("agent-")) assert.ok(content.includes("needs: [changes, fast-checks, agent-checks]"));
}
});
test("stable Rust, Agent and overall gates always inspect selected upstream results", () => {
for (const [name, mode, dependencies] of [["rust", "rust", ["fast-checks", "rust-fmt-clippy", "rust-test"]],
["agents", "agents", ["fast-checks", "agent-checks", "agent-rust", "agent-go", "agent-integration", "agent-java"]],
["ci", "all", ["rust", "agents", "frontend", "packages", "windows-win7-bundle", "duckdb-windows-driver", "nix-packaging"]]]) {
const content = job(name);
assert.match(content, /if: always\(\)/);
assert.ok(content.includes(`node .github/scripts/ci-gate.mjs ${mode}`));
assert.ok(content.includes("${{ toJSON(needs) }}"));
for (const dependency of dependencies) assert.match(content, new RegExp(`^ - ${dependency}$`, "m"));
}
});
test("every old Agent stage has an independent owner and Java packaging remains strict", () => {
assert.ok(job("agent-checks").includes("python3 -m unittest discover"));
assert.ok(job("agent-checks").includes("python3 scripts/validate_agents.py"));
assert.ok(job("agent-checks").includes("bump-agent-versions.test.mjs"));
assert.ok(job("agent-java").includes("./gradlew test shadowJar --continue"));
assert.ok(job("agent-java").includes("python3 scripts/validate_agent_jars.py"));
assert.ok(job("agent-rust").includes('cargo test --manifest-path "drivers/$DRIVER/Cargo.toml" --locked'));
assert.ok(job("agent-rust").includes("cargo build --manifest-path drivers/tdengine/Cargo.toml --locked --release --bin dbx-tdengine-driver"));
assert.ok(job("agent-go").includes("ci-agent-go.sh"));
assert.ok(job("agent-integration").includes("ci-agent-integration.sh"));
assert.ok(job("agent-integration").includes("Setup RocketMQ server Java"));
assert.ok(job("agent-integration").includes("if: matrix.scenario == 'rocketmq'"));
assert.ok(job("agent-integration").includes('java-version: "21"'));
for (const name of ["agent-rust", "agent-go", "agent-integration", "agent-java"]) assert.doesNotMatch(job(name), /continue-on-error: true/);
});
test("the planner uses the exact event base and preserves a single workflow cancellation scope", () => {
const changes = job("changes");
assert.ok(changes.includes("github.event.pull_request.base.sha || github.event.before"));
assert.ok(changes.includes("node .github/scripts/ci-plan.mjs"));
for (const flag of ["rust", "rust_full", "rust_matrix", "agents", "agent_go", "agent_rust", "agent_integration", "plan"]) {
assert.ok(changes.includes(`steps.plan.outputs.${flag}`));
}
assert.ok(workflow.includes("group: ${{ github.workflow }}-${{ github.ref }}"));
assert.ok(workflow.includes("cancel-in-progress: true"));
});
.github/workflows/ci.yml
+261 -406
View File
@@ -2,9 +2,9 @@ name: CI
on:
push:
branches: [main]
branches: [ main ]
pull_request:
branches: [main]
branches: [ main ]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -255,7 +255,9 @@ jobs:
"$DRIVER" < /dev/null
rust-fmt-clippy:
needs: changes
needs:
- changes
- fast-checks
if: needs.changes.outputs.rust == 'true'
runs-on: ubuntu-22.04
env:
@@ -264,8 +266,6 @@ jobs:
RUSTC_WRAPPER: sccache
# The fast lane skips only system font discovery while retaining the other default capabilities.
RUST_FEATURE_MODE: ${{ github.event_name == 'pull_request' && needs.changes.outputs.rust_full != 'true' && 'fast' || 'full' }}
RUST_FAST_FEATURES: dbx/duckdb-sidecar,dbx/dynamodb,dbx/mq-admin,dbx/sqlite-sqlcipher,dbx-core/duckdb-sidecar,dbx-core/dynamodb,dbx-core/mq-admin,dbx-core/sqlite-sqlcipher,dbx-web/duckdb-sidecar,dbx-web/dynamodb,dbx-web/mq-admin,dbx-web/sqlite-sqlcipher
RUST_FULL_FEATURES: dbx/duckdb-sidecar,dbx/dynamodb,dbx/mq-admin,dbx/sqlite-sqlcipher,dbx/system-fonts,dbx-core/duckdb-sidecar,dbx-core/dynamodb,dbx-core/mq-admin,dbx-core/sqlite-sqlcipher,dbx-core/system-fonts,dbx-web/duckdb-sidecar,dbx-web/dynamodb,dbx-web/mq-admin,dbx-web/sqlite-sqlcipher,dbx-web/system-fonts
# Fork PRs cannot read repository secrets, so retain the GHA backend for them.
SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }}
steps:
@@ -324,19 +324,9 @@ jobs:
# PR caches are large and branch-scoped; restore them from main without saving per-PR copies.
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Core architecture contracts
run: node --test scripts/core-architecture.test.mjs
- name: Cargo fmt check
run: cargo fmt --check
- name: Cargo clippy
run: |
if [ "$RUST_FEATURE_MODE" = "fast" ]; then
cargo clippy --workspace --locked --all-targets --no-default-features --features "$RUST_FAST_FEATURES" -- -D warnings
else
cargo clippy --workspace --locked --all-targets --no-default-features --features "$RUST_FULL_FEATURES" -- -D warnings
fi
run: |-
node .github/scripts/ci-rust.mjs clippy workspace "$RUST_FEATURE_MODE"
- name: Show sccache stats
if: always()
@@ -344,7 +334,9 @@ jobs:
run: ${SCCACHE_PATH} --show-stats
rust-test:
needs: changes
needs:
- changes
- fast-checks
if: needs.changes.outputs.rust == 'true'
runs-on: ubuntu-22.04
env:
@@ -355,10 +347,9 @@ jobs:
RUSTC_WRAPPER: sccache
# The fast lane skips only system font discovery while retaining the other default capabilities.
RUST_FEATURE_MODE: ${{ github.event_name == 'pull_request' && needs.changes.outputs.rust_full != 'true' && 'fast' || 'full' }}
RUST_FAST_FEATURES: dbx/duckdb-sidecar,dbx/dynamodb,dbx/mq-admin,dbx/sqlite-sqlcipher,dbx-core/duckdb-sidecar,dbx-core/dynamodb,dbx-core/mq-admin,dbx-core/sqlite-sqlcipher,dbx-web/duckdb-sidecar,dbx-web/dynamodb,dbx-web/mq-admin,dbx-web/sqlite-sqlcipher
RUST_FULL_FEATURES: dbx/duckdb-sidecar,dbx/dynamodb,dbx/mq-admin,dbx/sqlite-sqlcipher,dbx/system-fonts,dbx-core/duckdb-sidecar,dbx-core/dynamodb,dbx-core/mq-admin,dbx-core/sqlite-sqlcipher,dbx-core/system-fonts,dbx-web/duckdb-sidecar,dbx-web/dynamodb,dbx-web/mq-admin,dbx-web/sqlite-sqlcipher,dbx-web/system-fonts
# Fork PRs cannot read repository secrets, so retain the GHA backend for them.
SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }}
RUST_TEST_GROUP: ${{ matrix.group }}
steps:
- uses: actions/checkout@v5
@@ -366,6 +357,7 @@ jobs:
run: |
sudo apt-get update
sudo apt-get install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf libssl-dev libsecret-1-dev
if: matrix.group != 'foundation'
- name: Setup Rust
uses: dtolnay/rust-toolchain@1.97.1
@@ -416,33 +408,37 @@ jobs:
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Cargo test
run: |
if [ "$RUST_FEATURE_MODE" = "fast" ]; then
cargo test --workspace --locked --no-default-features --features "$RUST_FAST_FEATURES"
else
cargo test --workspace --locked --no-default-features --features "$RUST_FULL_FEATURES"
fi
run: |-
node .github/scripts/ci-rust.mjs test "$RUST_TEST_GROUP" "$RUST_FEATURE_MODE"
- name: Show sccache stats
if: always()
continue-on-error: true
run: ${SCCACHE_PATH} --show-stats
strategy:
fail-fast: false
max-parallel: 3
matrix: ${{ fromJSON(needs.changes.outputs.rust_matrix) }}
name: rust-test (${{ matrix.group }})
rust:
needs: [changes, rust-fmt-clippy, rust-test]
if: always() && needs.changes.outputs.rust == 'true'
needs:
- changes
- fast-checks
- rust-fmt-clippy
- rust-test
if: always()
runs-on: ubuntu-22.04
steps:
- name: Check Rust jobs
run: |
if [ "${{ needs.rust-fmt-clippy.result }}" != "success" ]; then
echo "rust-fmt-clippy result: ${{ needs.rust-fmt-clippy.result }}"
exit 1
fi
if [ "${{ needs.rust-test.result }}" != "success" ]; then
echo "rust-test result: ${{ needs.rust-test.result }}"
exit 1
fi
- uses: actions/checkout@v5
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: 22.13.0
- name: Check selected rust jobs
env:
NEEDS_JSON: ${{ toJSON(needs) }}
run: node .github/scripts/ci-gate.mjs rust
jdbc:
needs: changes
@@ -531,15 +527,26 @@ jobs:
outputs:
frontend: ${{ steps.filter.outputs.frontend }}
packages: ${{ steps.filter.outputs.packages }}
rust: ${{ steps.filter.outputs.rust }}
rust_full: ${{ steps.rust-mode.outputs.full }}
rust: ${{ steps.plan.outputs.rust }}
rust_full: ${{ steps.plan.outputs.rust_full }}
jdbc: ${{ steps.filter.outputs.jdbc }}
offline_jdbc: ${{ steps.filter.outputs.offline_jdbc }}
agents: ${{ steps.filter.outputs.agents }}
duckdb_windows: ${{ steps.filter.outputs.duckdb_windows }}
agents: ${{ steps.plan.outputs.agents }}
duckdb_windows: ${{ steps.filter.outputs.duckdb_windows == 'true' || steps.plan.outputs.duckdb_windows == 'true' }}
nix: ${{ steps.filter.outputs.nix }}
windows_win7_bundle: ${{ steps.filter.outputs.windows_win7_bundle }}
github_scripts: ${{ steps.filter.outputs.github_scripts }}
fast: ${{ steps.plan.outputs.fast }}
rust_matrix: ${{ steps.plan.outputs.rust_matrix }}
rust_groups_known: ${{ steps.plan.outputs.rust_groups_known }}
agent_java: ${{ steps.plan.outputs.agent_java }}
agent_go: ${{ steps.plan.outputs.agent_go }}
agent_rust: ${{ steps.plan.outputs.agent_rust }}
agent_integration: ${{ steps.plan.outputs.agent_integration }}
agent_go_changed: ${{ steps.plan.outputs.agent_go_changed }}
agent_rust_changed: ${{ steps.plan.outputs.agent_rust_changed }}
agent_integration_changed: ${{ steps.plan.outputs.agent_integration_changed }}
plan: ${{ steps.plan.outputs.plan }}
steps:
- uses: actions/checkout@v5
with:
@@ -659,37 +666,79 @@ jobs:
- '.github/workflows/ci.yml'
- 'apps/desktop/src/types/database.ts'
- 'crates/dbx-core/assets/database-drivers.manifest.json'
- name: Select Rust feature coverage
id: rust-mode
shell: bash
- name: Setup Rust
uses: dtolnay/rust-toolchain@1.97.1
- name: Plan dependency-aware CI jobs
id: plan
env:
BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
RUST_CHANGED: ${{ steps.filter.outputs.rust }}
run: |
full=false
if [ "$RUST_CHANGED" = "true" ] && [ "${{ github.event_name }}" = "push" ]; then
full=true
elif [ "$RUST_CHANGED" = "true" ]; then
if [ -z "$BASE_SHA" ] || echo "$BASE_SHA" | grep -Eq '^0+$'; then
BASE_SHA="HEAD~1"
fi
while IFS= read -r file; do
case "$file" in
Cargo.toml|Cargo.lock|rust-toolchain*|.github/workflows/ci.yml|*/Cargo.toml|vendor/*)
full=true
break
;;
esac
done < <(git diff --name-only "$BASE_SHA" HEAD -- Cargo.toml Cargo.lock 'rust-toolchain*' crates src-tauri vendor .github/workflows/ci.yml)
fi
echo "full=$full" >> "$GITHUB_OUTPUT"
AGENTS_CHANGED: ${{ steps.filter.outputs.agents }}
run: node .github/scripts/ci-plan.mjs
agents:
needs:
- changes
- fast-checks
- agent-checks
- agent-java
- agent-rust
- agent-go
- agent-integration
if: always()
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v5
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: 22.13.0
- name: Check selected agents jobs
env:
NEEDS_JSON: ${{ toJSON(needs) }}
run: node .github/scripts/ci-gate.mjs agents
fast-checks:
needs: changes
if: needs.changes.outputs.fast == 'true'
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v5
- name: Setup Rust
uses: dtolnay/rust-toolchain@1.97.1
with:
components: rustfmt
if: needs.changes.outputs.rust == 'true' || needs.changes.outputs.agent_rust_changed == 'true'
- name: Cargo fmt check
if: needs.changes.outputs.rust == 'true'
run: cargo fmt --check
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: 22.13.0
- name: CI planner and gate tests
run: node --test .github/scripts/ci-*.test.mjs
- name: Core architecture contracts
if: needs.changes.outputs.rust == 'true'
run: node --test scripts/core-architecture.test.mjs
- name: Setup pnpm
if: needs.changes.outputs.rust == 'true'
uses: pnpm/action-setup@v6
with:
version: 10.27.0
- name: Install generator dependencies
if: needs.changes.outputs.rust == 'true'
run: pnpm --filter dbx... install --frozen-lockfile --ignore-scripts
- name: Check generated connection types
if: needs.changes.outputs.rust == 'true'
run: node scripts/sync-connection-types.mjs --check
- name: Check workspace and standalone Cargo locks
env:
CI_PLAN: ${{ needs.changes.outputs.plan }}
run: node .github/scripts/ci-lockfiles.mjs
- name: Check grouped Rust feature coverage
if: needs.changes.outputs.rust == 'true' && needs.changes.outputs.rust_groups_known == 'true'
run: node .github/scripts/ci-rust-coverage.mjs
agent-checks:
needs: changes
if: needs.changes.outputs.agents == 'true'
runs-on: ubuntu-22.04
@@ -698,357 +747,163 @@ jobs:
working-directory: agents
steps:
- uses: actions/checkout@v5
- name: Setup Java
uses: actions/setup-java@v5
- name: Setup Node.js
uses: actions/setup-node@v6
with:
distribution: temurin
java-version: |
8
21
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: "1.25.x"
- name: Setup Rust
uses: dtolnay/rust-toolchain@1.97.1
- name: Setup Gradle
uses: gradle/actions/setup-gradle@v4
node-version: 22.13.0
- name: Install packaging tools
run: |
sudo apt-get update
sudo apt-get install -y zstd
- name: Agent script tests
run: |
python3 -m unittest discover -s scripts -p '*_test.py'
node --test ../.github/scripts/bump-agent-versions.test.mjs
- name: DuckDB driver tests
run: cargo test --manifest-path drivers/duckdb/Cargo.toml --locked
- name: TDengine driver tests
run: cargo test --manifest-path drivers/tdengine/Cargo.toml --locked
- name: Agent validation
run: python3 scripts/validate_agents.py
- name: Oracle native agent tests
run: go test ./...
working-directory: agents/drivers/oracle-go
- name: Xugu native agent tests
run: GONOSUMDB=gitee.com/XuguDB/go-xugu-driver go test ./...
working-directory: agents/drivers/xugu
- name: RabbitMQ native agent tests
run: go test ./...
working-directory: agents/drivers/rabbitmq
- name: RocketMQ native agent tests
run: go test -race ./...
working-directory: agents/drivers/rocketmq
- name: ZooKeeper native agent tests
run: go test -race ./...
working-directory: agents/drivers/zookeeper
- name: Cassandra native agent tests
run: go test ./...
working-directory: agents/drivers/cassandra-go
- name: Hive native agent tests
run: go test ./...
working-directory: agents/drivers/hive-go
- name: Vastbase native agent tests
run: go test ./...
working-directory: agents/drivers/vastbase-go
- name: Neo4j native agent tests
run: go test ./...
working-directory: agents/drivers/neo4j-go
- name: IoTDB native agent tests
run: go test ./...
working-directory: agents/drivers/iotdb
- name: Oracle native agent build
run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-oracle-linux-x64 .
working-directory: agents/drivers/oracle-go
- name: Xugu native agent build
run: GONOSUMDB=gitee.com/XuguDB/go-xugu-driver CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-xugu-linux-x64 .
working-directory: agents/drivers/xugu
- name: RabbitMQ native agent build
run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-rabbitmq-linux-x64 .
working-directory: agents/drivers/rabbitmq
- name: RocketMQ native agent build
run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-rocketmq-linux-x64 .
working-directory: agents/drivers/rocketmq
- name: ZooKeeper native agent cross-builds
shell: bash
working-directory: agents/drivers/zookeeper
agent-java:
needs: [changes, fast-checks, agent-checks]
if: needs.changes.outputs.agent_java == 'true'
runs-on: ubuntu-22.04
defaults:
run:
working-directory: agents
steps:
- uses: actions/checkout@v5
- name: Setup Java
uses: actions/setup-java@v5
with:
distribution: temurin
java-version: |-
8
21
- name: Setup Gradle
uses: gradle/actions/setup-gradle@v4
- name: Install packaging tools
run: |
set -euo pipefail
for target in darwin/arm64 darwin/amd64 linux/arm64 linux/amd64 windows/arm64 windows/amd64; do
IFS=/ read -r goos goarch <<< "$target"
output="/tmp/dbx-agent-zookeeper-${goos}-${goarch}"
[ "$goos" = windows ] && output="${output}.exe"
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags="-s -w" -o "$output" .
done
- name: ZooKeeper native agent integration tests
shell: bash
working-directory: agents/drivers/zookeeper
run: |
set -euo pipefail
for version in 3.4.14 3.5.5 3.7.0 3.9.5; do
name="dbx-zookeeper-${version//./-}"
docker rm -fv "$name" >/dev/null 2>&1 || true
docker run -d --name "$name" -p 2181:2181 "zookeeper:$version"
cleanup() {
docker rm -fv "$name" >/dev/null 2>&1 || true
}
trap cleanup EXIT
ready=false
for _ in $(seq 1 90); do
if timeout 1 bash -c '</dev/tcp/127.0.0.1/2181' >/dev/null 2>&1; then
ready=true
break
fi
sleep 2
done
if [ "$ready" != true ]; then
docker logs "$name"
exit 1
fi
DBX_ZOOKEEPER_TEST_CONNECT_STRING=127.0.0.1:2181 \
go test -run '^TestZooKeeperIntegration$' -count=1 ./...
if [ "$version" = "3.7.0" ]; then
DBX_ZOOKEEPER_TEST_CONNECT_STRING=127.0.0.1:2181 \
go test -run '^TestZooKeeperLargeChildrenIntegration$' -count=1 ./...
fi
cleanup
trap - EXIT
done
- name: ZooKeeper SASL DIGEST-MD5 integration test
shell: bash
working-directory: agents/drivers/zookeeper
run: |
set -euo pipefail
name="dbx-zookeeper-sasl-3-7-0"
jaas_file=$(mktemp)
cat > "$jaas_file" <<'EOF'
Server {
org.apache.zookeeper.server.auth.DigestLoginModule required
user_dbx="dbx-secret";
};
EOF
chmod 0644 "$jaas_file"
docker rm -fv "$name" >/dev/null 2>&1 || true
docker run -d --name "$name" -p 2181:2181 \
-v "$jaas_file:/conf/server-jaas.conf:ro" \
-e ZOO_CFG_EXTRA="authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider" \
-e JVMFLAGS="-Djava.security.auth.login.config=/conf/server-jaas.conf -Dzookeeper.sasl.serverconfig=Server -Dzookeeper.sessionRequireClientSASLAuth=true" \
zookeeper:3.7.0
cleanup() {
docker rm -fv "$name" >/dev/null 2>&1 || true
}
trap cleanup EXIT
ready=false
for _ in $(seq 1 90); do
if timeout 1 bash -c '</dev/tcp/127.0.0.1/2181' >/dev/null 2>&1; then
ready=true
break
fi
sleep 2
done
if [ "$ready" != true ]; then
docker logs "$name"
exit 1
fi
DBX_ZOOKEEPER_TEST_CONNECT_STRING=127.0.0.1:2181 \
DBX_ZOOKEEPER_TEST_AUTH_SCHEME=sasl_digest \
DBX_ZOOKEEPER_TEST_USERNAME=dbx \
DBX_ZOOKEEPER_TEST_PASSWORD=dbx-secret \
go test -run '^TestZooKeeperIntegration$' -count=1 ./...
- name: RocketMQ native agent integration tests
shell: bash
working-directory: agents/drivers/rocketmq
run: |
set -euo pipefail
for version in 4.9.8 5.3.1; do
./scripts/run-integration.sh "$version"
done
- name: Cassandra native agent build
run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-cassandra-linux-x64 .
working-directory: agents/drivers/cassandra-go
- name: Hive native agent build
run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-hive-linux-x64 .
working-directory: agents/drivers/hive-go
- name: Vastbase native agent build
run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-vastbase-linux-x64 .
working-directory: agents/drivers/vastbase-go
- name: Neo4j native agent build
run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-neo4j-linux-x64 .
working-directory: agents/drivers/neo4j-go
- name: IoTDB native agent build
run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-iotdb-linux-x64 .
working-directory: agents/drivers/iotdb
- name: TDengine native agent build
run: cargo build --manifest-path drivers/tdengine/Cargo.toml --locked --release --bin dbx-tdengine-driver
- name: TDengine native agent integration tests
shell: bash
run: |
set -euo pipefail
while IFS='|' read -r version image; do
name="dbx-tdengine-${version//./-}"
docker rm -fv "$name" >/dev/null 2>&1 || true
docker run -d --name "$name" \
--memory 4g \
--ulimit nofile=65535:65535 \
-p 6030:6030 \
-p 6041:6041 \
"$image"
cleanup() {
docker rm -fv "$name" >/dev/null 2>&1 || true
}
trap cleanup EXIT
ready=false
dnodes=""
for _ in $(seq 1 90); do
dnodes="$(docker exec "$name" taos -s 'SHOW DNODES' 2>/dev/null || true)"
if grep -Eq '\|[[:space:]]*ready[[:space:]]*\|' <<<"$dnodes"; then
ready=true
break
fi
sleep 2
done
if [ "$ready" != "true" ]; then
printf '%s\n' "$dnodes"
docker logs "$name"
exit 1
fi
TDENGINE_INTEGRATION=1 \
TDENGINE_TEST_HOST=127.0.0.1 \
TDENGINE_TEST_PORT=6041 \
cargo test --manifest-path drivers/tdengine/Cargo.toml --locked --test live -- --nocapture
cleanup
trap - EXIT
done <<'EOF'
2.4.0.14|tdengine/tdengine:2.4.0.14
2.6.0.34|tdengine/tdengine:2.6.0.34
3.0.7.1|tdengine/tdengine:3.0.7.1
3.3.6.13|tdengine/tdengine:3.3.6.13
3.4.2.2|tdengine/tsdb:3.4.2.2
EOF
- name: Cassandra native agent integration tests
shell: bash
working-directory: agents/drivers/cassandra-go
run: |
set -euo pipefail
for version in 3.11.19 5.0.6; do
name="dbx-cassandra-${version//./-}"
docker rm -fv "$name" >/dev/null 2>&1 || true
docker run -d --name "$name" \
-e CASSANDRA_CLUSTER_NAME="DBX Cassandra CI $version" \
-e CASSANDRA_DC=dc1 \
-e CASSANDRA_RACK=rack1 \
-e CASSANDRA_ENDPOINT_SNITCH=GossipingPropertyFileSnitch \
-e CASSANDRA_NUM_TOKENS=16 \
-e MAX_HEAP_SIZE=512M \
-e HEAP_NEWSIZE=100M \
-p 9042:9042 \
"cassandra:$version"
cleanup() {
docker rm -fv "$name" >/dev/null 2>&1 || true
}
trap cleanup EXIT
ready=false
for _ in $(seq 1 100); do
if docker exec "$name" cqlsh -e 'SELECT release_version FROM system.local' >/dev/null 2>&1; then
ready=true
break
fi
sleep 3
done
if [ "$ready" != "true" ]; then
docker logs "$name"
exit 1
fi
CASSANDRA_TEST_HOST=127.0.0.1 \
CASSANDRA_TEST_PORT=9042 \
go test -run '^TestCassandraIntegration$' -count=1 ./...
cleanup
trap - EXIT
done
- name: RabbitMQ native agent integration tests
shell: bash
working-directory: agents/drivers/rabbitmq
run: |
set -euo pipefail
for version in 3.13 4.3; do
name="dbx-rabbitmq-${version//./-}"
cookie="dbx-ci-${GITHUB_RUN_ID:-local}-${version//./-}"
# RabbitMQ is disposable in CI. Keep its data on a fresh tmpfs
# owned by the image's rabbitmq user so .erlang.cookie is readable.
docker rm -fv "$name" >/dev/null 2>&1 || true
docker run -d --name "$name" \
--user 999:999 \
--tmpfs /var/lib/rabbitmq:rw,exec,uid=999,gid=999,mode=700 \
-e RABBITMQ_DEFAULT_USER=dbx \
-e RABBITMQ_DEFAULT_PASS=dbx-password \
-e RABBITMQ_ERLANG_COOKIE="$cookie" \
-p 5672:5672 -p 15672:15672 \
"rabbitmq:${version}-management"
cleanup() {
docker rm -fv "$name" >/dev/null 2>&1 || true
}
trap cleanup EXIT
ready=false
for _ in $(seq 1 60); do
if docker exec "$name" rabbitmq-diagnostics -q check_running >/dev/null 2>&1 \
&& curl --fail --silent --noproxy '*' --user dbx:dbx-password \
http://127.0.0.1:15672/api/overview >/dev/null; then
ready=true
break
fi
sleep 2
done
if [ "$ready" != "true" ]; then
docker inspect "$name" --format 'image={{.Config.Image}} user={{.Config.User}} status={{.State.Status}} exit={{.State.ExitCode}}' || true
docker logs "$name"
exit 1
fi
RABBITMQ_INTEGRATION=1 \
RABBITMQ_USERNAME=dbx \
RABBITMQ_PASSWORD=dbx-password \
go test -run '^TestRabbitMQIntegration$' -count=1 ./...
cleanup
trap - EXIT
done
sudo apt-get update
sudo apt-get install -y zstd
- name: Java agent tests and packages
run: ./gradlew test shadowJar --continue
- name: Agent jar validation
run: python3 scripts/validate_agent_jars.py
agent-rust:
needs: [changes, fast-checks, agent-checks]
if: needs.changes.outputs.agent_rust_changed == 'true'
runs-on: ubuntu-22.04
defaults:
run:
working-directory: agents
steps:
- uses: actions/checkout@v5
- name: Setup Rust
uses: dtolnay/rust-toolchain@1.97.1
- name: Rust cache
uses: swatinem/rust-cache@v2
with:
workspaces: agents/drivers/${{ matrix.driver }} -> target
cache-on-failure: true
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Native Rust driver tests
env:
DRIVER: ${{ matrix.driver }}
run: cargo test --manifest-path "drivers/$DRIVER/Cargo.toml" --locked
- name: TDengine native agent build
if: matrix.driver == 'tdengine'
run: cargo build --manifest-path drivers/tdengine/Cargo.toml --locked --release --bin dbx-tdengine-driver
strategy:
fail-fast: false
max-parallel: 2
matrix: ${{ fromJSON(needs.changes.outputs.agent_rust) }}
agent-go:
needs: [changes, fast-checks, agent-checks]
if: needs.changes.outputs.agent_go_changed == 'true'
runs-on: ubuntu-22.04
defaults:
run:
working-directory: agents
steps:
- uses: actions/checkout@v5
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: 1.25.x
cache-dependency-path: agents/drivers/${{ matrix.driver }}/go.sum
- name: Native Go tests and target builds
env:
DRIVER: ${{ matrix.driver }}
BINARY: ${{ matrix.binary }}
RACE: ${{ matrix.race }}
run: bash ../.github/scripts/ci-agent-go.sh "$DRIVER" "$BINARY" "$RACE"
strategy:
fail-fast: false
max-parallel: 4
matrix: ${{ fromJSON(needs.changes.outputs.agent_go) }}
agent-integration:
needs: [changes, fast-checks, agent-checks]
if: needs.changes.outputs.agent_integration_changed == 'true'
runs-on: ubuntu-22.04
defaults:
run:
working-directory: agents
steps:
- uses: actions/checkout@v5
- name: Setup RocketMQ server Java
if: matrix.scenario == 'rocketmq'
uses: actions/setup-java@v5
with:
distribution: temurin
java-version: "21"
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: 1.25.x
cache-dependency-path: agents/drivers/${{ matrix.driver }}/go.sum
if: matrix.driver != 'tdengine'
- name: Setup Rust
uses: dtolnay/rust-toolchain@1.97.1
if: matrix.driver == 'tdengine'
- name: TDengine Rust cache
if: matrix.driver == 'tdengine'
uses: swatinem/rust-cache@v2
with:
workspaces: agents/drivers/tdengine -> target
cache-on-failure: true
save-if: ${{ github.ref == 'refs/heads/main' && matrix.version == '3.4.2.2' }}
- name: Run one live engine and version
env:
SCENARIO: ${{ matrix.scenario }}
VERSION: ${{ matrix.version }}
IMAGE: ${{ matrix.image || '' }}
run: bash ../.github/scripts/ci-agent-integration.sh "$SCENARIO" "$VERSION" "$IMAGE"
name: agent-integration (${{ matrix.scenario }}, ${{ matrix.version }})
strategy:
fail-fast: false
max-parallel: 4
matrix: ${{ fromJSON(needs.changes.outputs.agent_integration) }}
ci:
needs:
- changes
- fast-checks
- rust
- agents
- frontend
- github-scripts
- packages
- windows-win7-bundle
- duckdb-windows-driver
- jdbc
- offline-jdbc-release
- nix-packaging
if: always()
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v5
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: 22.13.0
- name: Check selected all jobs
env:
NEEDS_JSON: ${{ toJSON(needs) }}
run: node .github/scripts/ci-gate.mjs all