From 62aacbb2c9a784a4ac62aa8957a5ac6ebc211dfa Mon Sep 17 00:00:00 2001 From: "Adam Perry @ OpenAI" Date: Tue, 25 Aug 2026 23:32:10 +0000 Subject: [PATCH] Run exec-server compatibility tests under Bazel (#40736) ## What changed - Add a Bazel test rule that runs the shared Noise relay compatibility suite with current or packaged Codex binaries. - Cover both app-server/exec-server directions for the current build, release `0.149.1`, and the minimum supported release `0.145.0` on Linux x86-64. - Remove the standalone version-skew download scripts and the protocol constant they used. ## Testing The new `exec-server-current-version-test`, `exec-server-stable-release-test`, and `exec-server-minimum-release-test` targets exercise the compatibility suite. GitOrigin-RevId: 6b3b91eea8e05a38bec1fac128afc54f0d654014 --- bazel/rules/testing/compat/BUILD.bazel | 31 ++++++ .../compat/exec_server_compat_test.bzl | 47 ++++++++ .../testing/compat/exec_server_compat_test.rs | 20 ++-- codex-rs/exec-server-protocol/src/lib.rs | 3 - codex-rs/exec-server/BUILD.bazel | 17 +++ .../exec-server/testing/run_version_skew.sh | 100 ------------------ .../testing/run_version_skew_test.sh | 43 -------- codex-rs/exec-server/tests/relay.rs | 4 - 8 files changed, 104 insertions(+), 161 deletions(-) create mode 100644 bazel/rules/testing/compat/BUILD.bazel create mode 100644 bazel/rules/testing/compat/exec_server_compat_test.bzl rename codex-rs/exec-server/tests/relay/version_skew.rs => bazel/rules/testing/compat/exec_server_compat_test.rs (95%) delete mode 100755 codex-rs/exec-server/testing/run_version_skew.sh delete mode 100755 codex-rs/exec-server/testing/run_version_skew_test.sh diff --git a/bazel/rules/testing/compat/BUILD.bazel b/bazel/rules/testing/compat/BUILD.bazel new file mode 100644 index 0000000000..9f4bf9b0ba --- /dev/null +++ b/bazel/rules/testing/compat/BUILD.bazel @@ -0,0 +1,31 @@ +load("@rules_rust//rust:defs.bzl", "rust_test") + +package(default_visibility = ["//visibility:public"]) + +exports_files(["exec_server_compat_test.bzl"]) + +# exec_server_compat_test wraps this shared binary with current or released builds. +# Keep it manual so only wrapped compatibility tests enter the normal test graph. +rust_test( + name = "exec-server-compat-test-bin", + srcs = ["exec_server_compat_test.rs"], + crate_name = "exec_server_compat_test", + crate_root = "exec_server_compat_test.rs", + edition = "2024", + tags = ["manual"], + target_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + visibility = ["//codex-rs/exec-server:__pkg__"], + deps = [ + "//codex-rs/exec-server", + "//codex-rs/exec-server/tests/support", + "@crates//:anyhow", + "@crates//:pretty_assertions", + "@crates//:serde_json", + "@crates//:tempfile", + "@crates//:tokio", + "@crates//:wiremock", + ], +) diff --git a/bazel/rules/testing/compat/exec_server_compat_test.bzl b/bazel/rules/testing/compat/exec_server_compat_test.bzl new file mode 100644 index 0000000000..8e9f6c7533 --- /dev/null +++ b/bazel/rules/testing/compat/exec_server_compat_test.bzl @@ -0,0 +1,47 @@ +"""Bazel coverage for app-server and exec-server release compatibility.""" + +load("//:defs.bzl", "workspace_root_test") + +def exec_server_compat_test( + name, + comparison_binary = None, + current_binary = "//codex-rs/cli:codex", + release = None): + """Tests both executor directions against another Codex build or release. + + Args: + name: Name of the generated compatibility test target. + comparison_binary: Built Codex executable to compare with the current build. + current_binary: Built Codex executable representing the current version. + release: External release repository exposing `:codex` and `:package`. + """ + if (comparison_binary == None) == (release == None): + fail("exactly one of comparison_binary and release must be set") + + comparison = comparison_binary if release == None else release + "//:codex" + data = [] if release == None else [release + "//:package"] + comparison_alias = name + "-comparison-binary" + native.alias( + name = comparison_alias, + actual = comparison, + tags = ["manual"], + visibility = ["//visibility:private"], + ) + + workspace_root_test( + name = name, + args = ["--test-threads=1"], + data = data, + runfile_env = { + "//codex-rs/bwrap:bwrap": "CARGO_BIN_EXE_bwrap", + current_binary: "CODEX_TEST_CURRENT_CODEX", + ":" + comparison_alias: "CODEX_TEST_RELEASED_CODEX", + }, + tags = ["no-sandbox"], + target_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + test_bin = "//bazel/rules/testing/compat:exec-server-compat-test-bin", + workspace_root_marker = "//codex-rs/utils/cargo-bin:repo_root.marker", + ) diff --git a/codex-rs/exec-server/tests/relay/version_skew.rs b/bazel/rules/testing/compat/exec_server_compat_test.rs similarity index 95% rename from codex-rs/exec-server/tests/relay/version_skew.rs rename to bazel/rules/testing/compat/exec_server_compat_test.rs index 7543ae8c54..863d5a85e1 100644 --- a/codex-rs/exec-server/tests/relay/version_skew.rs +++ b/bazel/rules/testing/compat/exec_server_compat_test.rs @@ -3,7 +3,6 @@ use std::path::PathBuf; use std::process::Stdio; use std::sync::Arc; use std::sync::Mutex; -use std::time::Duration; use anyhow::Context; use anyhow::Result; @@ -11,6 +10,11 @@ use codex_exec_server::CODEX_EXEC_SERVER_NOISE_AUTH_TOKEN_ENV_VAR; use codex_exec_server::CODEX_EXEC_SERVER_NOISE_ENVIRONMENT_ID_ENV_VAR; use codex_exec_server::CODEX_EXEC_SERVER_NOISE_REGISTRY_URL_ENV_VAR; use codex_exec_server::CODEX_EXEC_SERVER_URL_ENV_VAR; +use codex_exec_server_test_support::relay::TEST_TIMEOUT as VERSION_SKEW_TIMEOUT; +use codex_exec_server_test_support::relay::accept_websocket; +use codex_exec_server_test_support::relay::assert_relay_data_is_encrypted; +use codex_exec_server_test_support::relay::proxy_relay_frames; +use codex_exec_server_test_support::relay::registered_executor_public_key; use pretty_assertions::assert_eq; use serde_json::Value; use serde_json::json; @@ -31,19 +35,13 @@ use wiremock::matchers::header; use wiremock::matchers::method; use wiremock::matchers::path; -use crate::relay_support::ENVIRONMENT_ID; -use crate::relay_support::EXECUTOR_REGISTRATION_ID; -use crate::relay_support::HARNESS_KEY_AUTHORIZATION; -use crate::relay_support::REGISTRY_TOKEN; -use crate::relay_support::accept_websocket; -use crate::relay_support::assert_relay_data_is_encrypted; -use crate::relay_support::proxy_relay_frames; -use crate::relay_support::registered_executor_public_key; - +const ENVIRONMENT_ID: &str = "env-noise-relay-test"; +const EXECUTOR_REGISTRATION_ID: &str = "registration-1"; +const HARNESS_KEY_AUTHORIZATION: &str = "harness-key-authorization"; +const REGISTRY_TOKEN: &str = "registry-token"; const RELEASED_CODEX_ENV_VAR: &str = "CODEX_TEST_RELEASED_CODEX"; const CURRENT_CODEX_ENV_VAR: &str = "CODEX_TEST_CURRENT_CODEX"; const EXECUTOR_MARKER_ENV_VAR: &str = "CODEX_EXECUTOR_VERSION_SKEW_MARKER"; -const VERSION_SKEW_TIMEOUT: Duration = Duration::from_secs(30); const EXPECTED_OUTPUT: &str = "executor-version-skew-ok"; #[tokio::test(flavor = "multi_thread", worker_threads = 4)] diff --git a/codex-rs/exec-server-protocol/src/lib.rs b/codex-rs/exec-server-protocol/src/lib.rs index bf6a057fe9..bdfaa3d145 100644 --- a/codex-rs/exec-server-protocol/src/lib.rs +++ b/codex-rs/exec-server-protocol/src/lib.rs @@ -9,6 +9,3 @@ pub use network_policy::*; pub use process_id::ProcessId; pub use protocol::*; pub use rpc::*; - -/// Oldest Codex release supported by the executor protocol. -pub const MINIMUM_SUPPORTED_CODEX_VERSION: &str = "0.145.0"; diff --git a/codex-rs/exec-server/BUILD.bazel b/codex-rs/exec-server/BUILD.bazel index d70b389698..d894273ced 100644 --- a/codex-rs/exec-server/BUILD.bazel +++ b/codex-rs/exec-server/BUILD.bazel @@ -1,4 +1,5 @@ load("//:defs.bzl", "codex_rust_crate") +load("//bazel/rules/testing/compat:exec_server_compat_test.bzl", "exec_server_compat_test") exports_files( ["src/proto/codex.exec_server.relay.v1.rs"], @@ -26,3 +27,19 @@ codex_rust_crate( test_tags = ["no-sandbox"], unit_test_args = ["--test-threads=1"], ) + +exec_server_compat_test( + name = "exec-server-current-version-test", + comparison_binary = "//codex-rs/cli:codex", + current_binary = "//codex-rs/cli:codex", +) + +exec_server_compat_test( + name = "exec-server-stable-release-test", + release = "@codex_release_0.149.1_linux_x86_64", +) + +exec_server_compat_test( + name = "exec-server-minimum-release-test", + release = "@codex_release_0.145.0_linux_x86_64", +) diff --git a/codex-rs/exec-server/testing/run_version_skew.sh b/codex-rs/exec-server/testing/run_version_skew.sh deleted file mode 100755 index dd52dc63eb..0000000000 --- a/codex-rs/exec-server/testing/run_version_skew.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" -minimum_supported_release="$( - sed -n 's/^pub const MINIMUM_SUPPORTED_CODEX_VERSION: &str = "\([^"]*\)";$/\1/p' \ - "${repo_root}/codex-rs/exec-server-protocol/src/lib.rs" -)" -: "${minimum_supported_release:?minimum supported Codex release is missing}" -release_directory="$(mktemp -d "${TMPDIR:-/tmp}/codex-exec-server-skew.XXXXXX")" -trap 'rm -rf "${release_directory:?}"' EXIT - -if [[ $# -eq 0 ]]; then - releases=(latest "${minimum_supported_release}") -else - releases=("$@") -fi - -case "$(uname -s):$(uname -m)" in - Darwin:arm64) target="aarch64-apple-darwin" ;; - Darwin:x86_64) target="x86_64-apple-darwin" ;; - Linux:aarch64 | Linux:arm64) target="aarch64-unknown-linux-musl" ;; - Linux:x86_64) target="x86_64-unknown-linux-musl" ;; - *) - echo "Unsupported platform: $(uname -s) $(uname -m)" >&2 - exit 1 - ;; -esac - -asset="codex-${target}.tar.gz" - -if [[ "$(uname -s)" == "Linux" ]] \ - && command -v apt-get >/dev/null \ - && command -v dpkg-deb >/dev/null \ - && command -v dpkg-architecture >/dev/null \ - && ! pkg-config --exists alsa glib-2.0 gobject-2.0 gio-2.0; then - # Debian-like hosts can download development metadata without sudo. The - # temporary sysroot still relies on runtime libraries installed on the host. - voice_package_directory="${release_directory}/voice-packages" - voice_sysroot="${release_directory}/voice-sysroot" - mkdir -p "${voice_package_directory}" "${voice_sysroot}" - ( - cd "${voice_package_directory}" - apt-get download libasound2-dev libglib2.0-dev libcap-dev libffi-dev libmount-dev \ - libpcre2-dev libselinux1-dev zlib1g-dev libblkid-dev libsepol-dev - ) - for package in "${voice_package_directory}"/*.deb; do - dpkg-deb --extract "${package}" "${voice_sysroot}" - done - - voice_multiarch="$(dpkg-architecture -qDEB_HOST_MULTIARCH)" - voice_library_directory="${voice_sysroot}/usr/lib/${voice_multiarch}" - for library in "${voice_library_directory}"/*.so; do - [[ -L "${library}" ]] || continue - library_target="$(readlink "${library}")" - if [[ ! -e "${voice_library_directory}/${library_target}" && -e "/usr/lib/${voice_multiarch}/${library_target}" ]]; then - ln -s "/usr/lib/${voice_multiarch}/${library_target}" "${voice_library_directory}/${library_target}" - fi - done - - export PKG_CONFIG_PATH="${voice_library_directory}/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" - export PKG_CONFIG_SYSROOT_DIR="${voice_sysroot}" -fi - -cd "${repo_root}/codex-rs" -cargo build -p codex-cli --bin codex -export CODEX_TEST_CURRENT_CODEX="${CARGO_TARGET_DIR:-${repo_root}/codex-rs/target}/debug/codex" - -echo "Testing current Codex compatibility through authenticated Noise" -export CODEX_TEST_RELEASED_CODEX="${CODEX_TEST_CURRENT_CODEX}" -just test -p codex-exec-server --test relay version_skew --test-threads 1 - -tested_release_version="" -for release in "${releases[@]}"; do - release="${release#rust-v}" - if [[ "${release}" == "${tested_release_version}" ]]; then - echo "Skipping Codex ${release}; this release was already tested" - continue - fi - - if [[ "${release}" == "latest" ]]; then - release_url="https://github.com/openai/codex/releases/latest/download/${asset}" - else - release_url="https://github.com/openai/codex/releases/download/rust-v${release}/${asset}" - fi - - binary_directory="${release_directory}/${release}" - mkdir -p "${binary_directory}" - echo "Downloading released Codex from ${release_url}" - curl -fsSL "${release_url}" -o "${binary_directory}/${asset}" - tar -xzf "${binary_directory}/${asset}" -C "${binary_directory}" - - export CODEX_TEST_RELEASED_CODEX="${binary_directory}/codex-${target}" - release_output="$("${CODEX_TEST_RELEASED_CODEX}" --version)" - echo "${release_output}" - tested_release_version="${release_output##* }" - - just test -p codex-exec-server --test relay version_skew --test-threads 1 -done diff --git a/codex-rs/exec-server/testing/run_version_skew_test.sh b/codex-rs/exec-server/testing/run_version_skew_test.sh deleted file mode 100755 index 109a61cefa..0000000000 --- a/codex-rs/exec-server/testing/run_version_skew_test.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -script_path="${script_directory}/run_version_skew.sh" -fake_bin="$(mktemp -d "${TMPDIR:-/tmp}/codex-version-skew-test.XXXXXX")" -trap 'rm -rf "${fake_bin:?}"' EXIT - -for tool in bash dirname mkdir mktemp rm sed; do - ln -s "$(command -v "${tool}")" "${fake_bin}/${tool}" -done - -printf '%s\n' \ - '#!/usr/bin/env bash' \ - 'case "${1:-}" in' \ - ' -s) printf "Linux\\n" ;;' \ - ' -m) printf "x86_64\\n" ;;' \ - ' *) exit 1 ;;' \ - 'esac' >"${fake_bin}/uname" -chmod +x "${fake_bin}/uname" - -printf '%s\n' \ - '#!/usr/bin/env bash' \ - 'exit 1' >"${fake_bin}/pkg-config" -chmod +x "${fake_bin}/pkg-config" - -printf '%s\n' \ - '#!/usr/bin/env bash' \ - 'printf "reached cargo\\n" >&2' \ - 'exit 97' >"${fake_bin}/cargo" -chmod +x "${fake_bin}/cargo" - -set +e -output="$(PATH="${fake_bin}" "${script_path}" latest 2>&1)" -status=$? -set -e - -if [[ "${status}" -ne 97 || "${output}" != *"reached cargo"* ]]; then - printf '%s\n' "expected non-Debian Linux without voice metadata to reach cargo" >&2 - printf 'status: %s\n%s\n' "${status}" "${output}" >&2 - exit 1 -fi diff --git a/codex-rs/exec-server/tests/relay.rs b/codex-rs/exec-server/tests/relay.rs index 3201538366..f7dc181ed6 100644 --- a/codex-rs/exec-server/tests/relay.rs +++ b/codex-rs/exec-server/tests/relay.rs @@ -1,9 +1,5 @@ mod common; -#[cfg(unix)] -#[path = "relay/version_skew.rs"] -mod version_skew; - #[path = "common/relay.rs"] mod relay_support;