Run exec-server compatibility tests under Bazel (#40736)

## What changed

- Add a Bazel test rule that runs the shared Noise relay compatibility suite with current or packaged Codex binaries.
- Cover both app-server/exec-server directions for the current build, release `0.149.1`, and the minimum supported release `0.145.0` on Linux x86-64.
- Remove the standalone version-skew download scripts and the protocol constant they used.

## Testing

The new `exec-server-current-version-test`, `exec-server-stable-release-test`, and `exec-server-minimum-release-test` targets exercise the compatibility suite.

GitOrigin-RevId: 6b3b91eea8e05a38bec1fac128afc54f0d654014

bazel/rules/testing/compat/BUILD.bazel

@@ -0,0 +1,31 @@
+load("@rules_rust//rust:defs.bzl", "rust_test")
+
+package(default_visibility = ["//visibility:public"])
+
+exports_files(["exec_server_compat_test.bzl"])
+
+# exec_server_compat_test wraps this shared binary with current or released builds.
+# Keep it manual so only wrapped compatibility tests enter the normal test graph.
+rust_test(
+    name = "exec-server-compat-test-bin",
+    srcs = ["exec_server_compat_test.rs"],
+    crate_name = "exec_server_compat_test",
+    crate_root = "exec_server_compat_test.rs",
+    edition = "2024",
+    tags = ["manual"],
+    target_compatible_with = [
+        "@platforms//cpu:x86_64",
+        "@platforms//os:linux",
+    ],
+    visibility = ["//codex-rs/exec-server:__pkg__"],
+    deps = [
+        "//codex-rs/exec-server",
+        "//codex-rs/exec-server/tests/support",
+        "@crates//:anyhow",
+        "@crates//:pretty_assertions",
+        "@crates//:serde_json",
+        "@crates//:tempfile",
+        "@crates//:tokio",
+        "@crates//:wiremock",
+    ],
+)

bazel/rules/testing/compat/exec_server_compat_test.bzl

@@ -0,0 +1,47 @@
+"""Bazel coverage for app-server and exec-server release compatibility."""
+
+load("//:defs.bzl", "workspace_root_test")
+
+def exec_server_compat_test(
+        name,
+        comparison_binary = None,
+        current_binary = "//codex-rs/cli:codex",
+        release = None):
+    """Tests both executor directions against another Codex build or release.
+
+    Args:
+        name: Name of the generated compatibility test target.
+        comparison_binary: Built Codex executable to compare with the current build.
+        current_binary: Built Codex executable representing the current version.
+        release: External release repository exposing `:codex` and `:package`.
+    """
+    if (comparison_binary == None) == (release == None):
+        fail("exactly one of comparison_binary and release must be set")
+
+    comparison = comparison_binary if release == None else release + "//:codex"
+    data = [] if release == None else [release + "//:package"]
+    comparison_alias = name + "-comparison-binary"
+    native.alias(
+        name = comparison_alias,
+        actual = comparison,
+        tags = ["manual"],
+        visibility = ["//visibility:private"],
+    )
+
+    workspace_root_test(
+        name = name,
+        args = ["--test-threads=1"],
+        data = data,
+        runfile_env = {
+            "//codex-rs/bwrap:bwrap": "CARGO_BIN_EXE_bwrap",
+            current_binary: "CODEX_TEST_CURRENT_CODEX",
+            ":" + comparison_alias: "CODEX_TEST_RELEASED_CODEX",
+        },
+        tags = ["no-sandbox"],
+        target_compatible_with = [
+            "@platforms//cpu:x86_64",
+            "@platforms//os:linux",
+        ],
+        test_bin = "//bazel/rules/testing/compat:exec-server-compat-test-bin",
+        workspace_root_marker = "//codex-rs/utils/cargo-bin:repo_root.marker",
+    )

bazel/rules/testing/compat/exec_server_compat_test.rs

@@ -3,7 +3,6 @@ use std::path::PathBuf;
 use std::process::Stdio;
 use std::sync::Arc;
 use std::sync::Mutex;
-use std::time::Duration;
 
 use anyhow::Context;
 use anyhow::Result;
@@ -11,6 +10,11 @@ use codex_exec_server::CODEX_EXEC_SERVER_NOISE_AUTH_TOKEN_ENV_VAR;
 use codex_exec_server::CODEX_EXEC_SERVER_NOISE_ENVIRONMENT_ID_ENV_VAR;
 use codex_exec_server::CODEX_EXEC_SERVER_NOISE_REGISTRY_URL_ENV_VAR;
 use codex_exec_server::CODEX_EXEC_SERVER_URL_ENV_VAR;
+use codex_exec_server_test_support::relay::TEST_TIMEOUT as VERSION_SKEW_TIMEOUT;
+use codex_exec_server_test_support::relay::accept_websocket;
+use codex_exec_server_test_support::relay::assert_relay_data_is_encrypted;
+use codex_exec_server_test_support::relay::proxy_relay_frames;
+use codex_exec_server_test_support::relay::registered_executor_public_key;
 use pretty_assertions::assert_eq;
 use serde_json::Value;
 use serde_json::json;
@@ -31,19 +35,13 @@ use wiremock::matchers::header;
 use wiremock::matchers::method;
 use wiremock::matchers::path;
 
-use crate::relay_support::ENVIRONMENT_ID;
-use crate::relay_support::EXECUTOR_REGISTRATION_ID;
-use crate::relay_support::HARNESS_KEY_AUTHORIZATION;
-use crate::relay_support::REGISTRY_TOKEN;
-use crate::relay_support::accept_websocket;
-use crate::relay_support::assert_relay_data_is_encrypted;
-use crate::relay_support::proxy_relay_frames;
-use crate::relay_support::registered_executor_public_key;
-
+const ENVIRONMENT_ID: &str = "env-noise-relay-test";
+const EXECUTOR_REGISTRATION_ID: &str = "registration-1";
+const HARNESS_KEY_AUTHORIZATION: &str = "harness-key-authorization";
+const REGISTRY_TOKEN: &str = "registry-token";
 const RELEASED_CODEX_ENV_VAR: &str = "CODEX_TEST_RELEASED_CODEX";
 const CURRENT_CODEX_ENV_VAR: &str = "CODEX_TEST_CURRENT_CODEX";
 const EXECUTOR_MARKER_ENV_VAR: &str = "CODEX_EXECUTOR_VERSION_SKEW_MARKER";
-const VERSION_SKEW_TIMEOUT: Duration = Duration::from_secs(30);
 const EXPECTED_OUTPUT: &str = "executor-version-skew-ok";
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]

codex-rs/exec-server-protocol/src/lib.rs

@@ -9,6 +9,3 @@ pub use network_policy::*;
 pub use process_id::ProcessId;
 pub use protocol::*;
 pub use rpc::*;
-
-/// Oldest Codex release supported by the executor protocol.
-pub const MINIMUM_SUPPORTED_CODEX_VERSION: &str = "0.145.0";

codex-rs/exec-server/BUILD.bazel

@@ -1,4 +1,5 @@
 load("//:defs.bzl", "codex_rust_crate")
+load("//bazel/rules/testing/compat:exec_server_compat_test.bzl", "exec_server_compat_test")
 
 exports_files(
     ["src/proto/codex.exec_server.relay.v1.rs"],
@@ -26,3 +27,19 @@ codex_rust_crate(
     test_tags = ["no-sandbox"],
     unit_test_args = ["--test-threads=1"],
 )
+
+exec_server_compat_test(
+    name = "exec-server-current-version-test",
+    comparison_binary = "//codex-rs/cli:codex",
+    current_binary = "//codex-rs/cli:codex",
+)
+
+exec_server_compat_test(
+    name = "exec-server-stable-release-test",
+    release = "@codex_release_0.149.1_linux_x86_64",
+)
+
+exec_server_compat_test(
+    name = "exec-server-minimum-release-test",
+    release = "@codex_release_0.145.0_linux_x86_64",
+)

codex-rs/exec-server/testing/run_version_skew.sh

@@ -1,100 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
-minimum_supported_release="$(
-  sed -n 's/^pub const MINIMUM_SUPPORTED_CODEX_VERSION: &str = "\([^"]*\)";$/\1/p' \
-    "${repo_root}/codex-rs/exec-server-protocol/src/lib.rs"
-)"
-: "${minimum_supported_release:?minimum supported Codex release is missing}"
-release_directory="$(mktemp -d "${TMPDIR:-/tmp}/codex-exec-server-skew.XXXXXX")"
-trap 'rm -rf "${release_directory:?}"' EXIT
-
-if [[ $# -eq 0 ]]; then
-  releases=(latest "${minimum_supported_release}")
-else
-  releases=("$@")
-fi
-
-case "$(uname -s):$(uname -m)" in
-  Darwin:arm64) target="aarch64-apple-darwin" ;;
-  Darwin:x86_64) target="x86_64-apple-darwin" ;;
-  Linux:aarch64 | Linux:arm64) target="aarch64-unknown-linux-musl" ;;
-  Linux:x86_64) target="x86_64-unknown-linux-musl" ;;
-  *)
-    echo "Unsupported platform: $(uname -s) $(uname -m)" >&2
-    exit 1
-    ;;
-esac
-
-asset="codex-${target}.tar.gz"
-
-if [[ "$(uname -s)" == "Linux" ]] \
-  && command -v apt-get >/dev/null \
-  && command -v dpkg-deb >/dev/null \
-  && command -v dpkg-architecture >/dev/null \
-  && ! pkg-config --exists alsa glib-2.0 gobject-2.0 gio-2.0; then
-  # Debian-like hosts can download development metadata without sudo. The
-  # temporary sysroot still relies on runtime libraries installed on the host.
-  voice_package_directory="${release_directory}/voice-packages"
-  voice_sysroot="${release_directory}/voice-sysroot"
-  mkdir -p "${voice_package_directory}" "${voice_sysroot}"
-  (
-    cd "${voice_package_directory}"
-    apt-get download libasound2-dev libglib2.0-dev libcap-dev libffi-dev libmount-dev \
-      libpcre2-dev libselinux1-dev zlib1g-dev libblkid-dev libsepol-dev
-  )
-  for package in "${voice_package_directory}"/*.deb; do
-    dpkg-deb --extract "${package}" "${voice_sysroot}"
-  done
-
-  voice_multiarch="$(dpkg-architecture -qDEB_HOST_MULTIARCH)"
-  voice_library_directory="${voice_sysroot}/usr/lib/${voice_multiarch}"
-  for library in "${voice_library_directory}"/*.so; do
-    [[ -L "${library}" ]] || continue
-    library_target="$(readlink "${library}")"
-    if [[ ! -e "${voice_library_directory}/${library_target}" && -e "/usr/lib/${voice_multiarch}/${library_target}" ]]; then
-      ln -s "/usr/lib/${voice_multiarch}/${library_target}" "${voice_library_directory}/${library_target}"
-    fi
-  done
-
-  export PKG_CONFIG_PATH="${voice_library_directory}/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}"
-  export PKG_CONFIG_SYSROOT_DIR="${voice_sysroot}"
-fi
-
-cd "${repo_root}/codex-rs"
-cargo build -p codex-cli --bin codex
-export CODEX_TEST_CURRENT_CODEX="${CARGO_TARGET_DIR:-${repo_root}/codex-rs/target}/debug/codex"
-
-echo "Testing current Codex compatibility through authenticated Noise"
-export CODEX_TEST_RELEASED_CODEX="${CODEX_TEST_CURRENT_CODEX}"
-just test -p codex-exec-server --test relay version_skew --test-threads 1
-
-tested_release_version=""
-for release in "${releases[@]}"; do
-  release="${release#rust-v}"
-  if [[ "${release}" == "${tested_release_version}" ]]; then
-    echo "Skipping Codex ${release}; this release was already tested"
-    continue
-  fi
-
-  if [[ "${release}" == "latest" ]]; then
-    release_url="https://github.com/openai/codex/releases/latest/download/${asset}"
-  else
-    release_url="https://github.com/openai/codex/releases/download/rust-v${release}/${asset}"
-  fi
-
-  binary_directory="${release_directory}/${release}"
-  mkdir -p "${binary_directory}"
-  echo "Downloading released Codex from ${release_url}"
-  curl -fsSL "${release_url}" -o "${binary_directory}/${asset}"
-  tar -xzf "${binary_directory}/${asset}" -C "${binary_directory}"
-
-  export CODEX_TEST_RELEASED_CODEX="${binary_directory}/codex-${target}"
-  release_output="$("${CODEX_TEST_RELEASED_CODEX}" --version)"
-  echo "${release_output}"
-  tested_release_version="${release_output##* }"
-
-  just test -p codex-exec-server --test relay version_skew --test-threads 1
-done

codex-rs/exec-server/testing/run_version_skew_test.sh

@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-script_path="${script_directory}/run_version_skew.sh"
-fake_bin="$(mktemp -d "${TMPDIR:-/tmp}/codex-version-skew-test.XXXXXX")"
-trap 'rm -rf "${fake_bin:?}"' EXIT
-
-for tool in bash dirname mkdir mktemp rm sed; do
-  ln -s "$(command -v "${tool}")" "${fake_bin}/${tool}"
-done
-
-printf '%s\n' \
-  '#!/usr/bin/env bash' \
-  'case "${1:-}" in' \
-  '  -s) printf "Linux\\n" ;;' \
-  '  -m) printf "x86_64\\n" ;;' \
-  '  *) exit 1 ;;' \
-  'esac' >"${fake_bin}/uname"
-chmod +x "${fake_bin}/uname"
-
-printf '%s\n' \
-  '#!/usr/bin/env bash' \
-  'exit 1' >"${fake_bin}/pkg-config"
-chmod +x "${fake_bin}/pkg-config"
-
-printf '%s\n' \
-  '#!/usr/bin/env bash' \
-  'printf "reached cargo\\n" >&2' \
-  'exit 97' >"${fake_bin}/cargo"
-chmod +x "${fake_bin}/cargo"
-
-set +e
-output="$(PATH="${fake_bin}" "${script_path}" latest 2>&1)"
-status=$?
-set -e
-
-if [[ "${status}" -ne 97 || "${output}" != *"reached cargo"* ]]; then
-  printf '%s\n' "expected non-Debian Linux without voice metadata to reach cargo" >&2
-  printf 'status: %s\n%s\n' "${status}" "${output}" >&2
-  exit 1
-fi

codex-rs/exec-server/tests/relay.rs

@@ -1,9 +1,5 @@
 mod common;
 
-#[cfg(unix)]
-#[path = "relay/version_skew.rs"]
-mod version_skew;
-
 #[path = "common/relay.rs"]
 mod relay_support;