Route packaged rg through exec server runtime

This commit is contained in:
jif-oai
2026-07-09 23:57:56 +01:00
parent 6ced1ac5eb
commit e67c08dd51
16 changed files with 361 additions and 14 deletions
+9
View File
@@ -2910,6 +2910,7 @@ dependencies = [
"codex-exec-server-protocol",
"codex-file-system",
"codex-http-client",
"codex-install-context",
"codex-network-proxy",
"codex-otel",
"codex-protocol",
@@ -3811,6 +3812,14 @@ dependencies = [
"zeroize",
]
[[package]]
name = "codex-rg"
version = "0.0.0"
dependencies = [
"pretty_assertions",
"tempfile",
]
[[package]]
name = "codex-rmcp-client"
version = "0.0.0"
+1
View File
@@ -83,6 +83,7 @@ members = [
"rollout-trace",
"rmcp-client",
"responses-api-proxy",
"rg-shim",
"response-debug-context",
"sandboxing",
"stdio-to-uds",
+1
View File
@@ -18,6 +18,7 @@ bytes = { workspace = true }
clatter = { workspace = true }
codex-api = { workspace = true }
codex-http-client = { workspace = true }
codex-install-context = { workspace = true }
codex-exec-server-protocol = { workspace = true }
codex-file-system = { workspace = true }
codex-network-proxy = { workspace = true }
+21 -11
View File
@@ -234,8 +234,11 @@ impl LocalProcess {
params: ExecParams,
) -> Result<(ExecResponse, watch::Sender<u64>, ExecProcessEventLog), JSONRPCErrorError> {
let process_id = params.process_id.clone();
let prepared =
prepare_exec_request(&params, child_env(&params), self.runtime_paths.as_ref())?;
let prepared = prepare_exec_request(
&params,
child_env(&params, self.runtime_paths.as_ref()),
self.runtime_paths.as_ref(),
)?;
let (program, args) = prepared
.command
.split_first()
@@ -573,14 +576,21 @@ impl LocalProcess {
}
}
fn child_env(params: &ExecParams) -> HashMap<String, String> {
let Some(env_policy) = &params.env_policy else {
return params.env.clone();
fn child_env(
params: &ExecParams,
runtime_paths: Option<&ExecServerRuntimePaths>,
) -> HashMap<String, String> {
let mut env = if let Some(env_policy) = &params.env_policy {
let policy = shell_environment_policy(env_policy);
let mut env = shell_environment::create_env(&policy, /*thread_id*/ None);
env.extend(params.env.clone());
env
} else {
params.env.clone()
};
let policy = shell_environment_policy(env_policy);
let mut env = shell_environment::create_env(&policy, /*thread_id*/ None);
env.extend(params.env.clone());
if let Some(runtime_paths) = runtime_paths {
runtime_paths.apply_to_env(&mut env);
}
env
}
@@ -1090,7 +1100,7 @@ mod tests {
let params = test_exec_params(HashMap::from([("ONLY_THIS".to_string(), "1".to_string())]));
assert_eq!(
child_env(&params),
child_env(&params, /*runtime_paths*/ None),
HashMap::from([("ONLY_THIS".to_string(), "1".to_string())])
);
}
@@ -1117,7 +1127,7 @@ mod tests {
expected.insert("PATHEXT".to_string(), ".COM;.EXE;.BAT;.CMD".to_string());
}
assert_eq!(child_env(&params), expected);
assert_eq!(child_env(&params, /*runtime_paths*/ None), expected);
}
#[tokio::test]
+46 -1
View File
@@ -1,5 +1,8 @@
use std::collections::HashMap;
use std::path::Path;
use std::path::PathBuf;
use codex_install_context::InstallContext;
use codex_utils_absolute_path::AbsolutePathBuf;
/// Runtime paths needed by exec-server child processes.
@@ -10,6 +13,7 @@ pub struct ExecServerRuntimePaths {
/// Path to the Linux sandbox helper alias used when the platform sandbox
/// needs to re-enter Codex by argv0.
pub codex_linux_sandbox_exe: Option<AbsolutePathBuf>,
package_path_dir: Option<AbsolutePathBuf>,
}
impl ExecServerRuntimePaths {
@@ -30,14 +34,55 @@ impl ExecServerRuntimePaths {
codex_self_exe: PathBuf,
codex_linux_sandbox_exe: Option<PathBuf>,
) -> std::io::Result<Self> {
let codex_self_exe = absolute_path(codex_self_exe)?;
let package_path_dir = InstallContext::from_exe(
cfg!(target_os = "macos"),
Some(codex_self_exe.as_path()),
/*method_override*/ None,
)
.package_layout
.and_then(|layout| layout.path_dir);
Ok(Self {
codex_self_exe: absolute_path(codex_self_exe)?,
codex_self_exe,
codex_linux_sandbox_exe: codex_linux_sandbox_exe.map(absolute_path).transpose()?,
package_path_dir,
})
}
pub(crate) fn apply_to_env(&self, env: &mut HashMap<String, String>) {
if let Some(package_path_dir) = &self.package_path_dir {
prepend_package_path(env, package_path_dir.as_path());
}
}
}
fn prepend_package_path(env: &mut HashMap<String, String>, package_path_dir: &Path) {
let path_key = env
.keys()
.find(|key| *key == "PATH" || (cfg!(windows) && key.eq_ignore_ascii_case("PATH")))
.cloned();
let Some(path_key) = path_key else {
return;
};
let Some(current_path) = env.get(&path_key) else {
return;
};
if current_path.is_empty() {
return;
}
let entries = std::iter::once(package_path_dir.to_path_buf())
.chain(std::env::split_paths(current_path).filter(|entry| entry != package_path_dir));
if let Ok(path) = std::env::join_paths(entries) {
env.insert(path_key, path.to_string_lossy().into_owned());
}
}
fn absolute_path(path: PathBuf) -> std::io::Result<AbsolutePathBuf> {
AbsolutePathBuf::from_absolute_path(path.as_path())
.map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err))
}
#[cfg(test)]
#[path = "runtime_paths_tests.rs"]
mod tests;
@@ -0,0 +1,68 @@
use super::ExecServerRuntimePaths;
use super::prepend_package_path;
use pretty_assertions::assert_eq;
use std::collections::HashMap;
use std::fs;
use tempfile::TempDir;
#[test]
fn discovers_package_path_from_codex_executable() {
let temp_dir = TempDir::new().expect("temp dir");
let package_dir = temp_dir.path();
let bin_dir = package_dir.join("bin");
let package_path_dir = package_dir.join("codex-path");
fs::create_dir(&bin_dir).expect("bin dir");
fs::create_dir(&package_path_dir).expect("package path dir");
fs::write(package_dir.join("codex-package.json"), "{}").expect("metadata");
let codex_exe = bin_dir.join(if cfg!(windows) { "codex.exe" } else { "codex" });
fs::write(&codex_exe, "codex").expect("codex executable");
let runtime_paths =
ExecServerRuntimePaths::new(codex_exe, /*codex_linux_sandbox_exe*/ None)
.expect("runtime paths");
let package_path_dir = fs::canonicalize(package_path_dir).expect("canonical package path");
assert_eq!(
runtime_paths
.package_path_dir
.as_ref()
.map(|path| path.as_path()),
Some(package_path_dir.as_path())
);
}
#[test]
fn package_path_is_prepended_and_deduplicated() {
let package_path = std::path::PathBuf::from("package-bin");
let other_path = std::path::PathBuf::from("system-bin");
let initial_path = std::env::join_paths([
other_path.as_path(),
package_path.as_path(),
package_path.as_path(),
])
.expect("initial path");
let mut env = HashMap::from([(
"PATH".to_string(),
initial_path.to_string_lossy().into_owned(),
)]);
prepend_package_path(&mut env, &package_path);
let path = env.get("PATH").expect("PATH");
assert_eq!(
std::env::split_paths(path).collect::<Vec<_>>(),
vec![package_path, other_path]
);
}
#[test]
fn package_path_preserves_missing_path() {
let mut env = HashMap::from([("ONLY_THIS".to_string(), "1".to_string())]);
prepend_package_path(&mut env, std::path::Path::new("package-bin"));
assert_eq!(
env,
HashMap::from([("ONLY_THIS".to_string(), "1".to_string())])
);
}
+6
View File
@@ -0,0 +1,6 @@
load("//:defs.bzl", "codex_rust_crate")
codex_rust_crate(
name = "rg-shim",
crate_name = "codex_rg",
)
+16
View File
@@ -0,0 +1,16 @@
[package]
name = "codex-rg"
version.workspace = true
edition.workspace = true
license.workspace = true
[[bin]]
name = "codex-rg"
path = "src/main.rs"
[lints]
workspace = true
[dev-dependencies]
pretty_assertions = { workspace = true }
tempfile = { workspace = true }
+81
View File
@@ -0,0 +1,81 @@
use std::ffi::OsStr;
use std::io;
use std::path::Path;
use std::path::PathBuf;
use std::process::Command;
const PACKAGE_METADATA_FILENAME: &str = "codex-package.json";
const PATH_DIRNAME: &str = "codex-path";
const RESOURCES_DIRNAME: &str = "codex-resources";
fn main() {
let result = std::env::current_exe()
.and_then(|current_exe| real_rg_path(&current_exe))
.and_then(run_real_rg);
match result {
Ok(exit_code) => std::process::exit(exit_code),
Err(error) => {
eprintln!("codex-rg: {error}");
std::process::exit(127);
}
}
}
fn real_rg_path(current_exe: &Path) -> io::Result<PathBuf> {
let path_dir = current_exe.parent().ok_or_else(|| {
io::Error::new(
io::ErrorKind::NotFound,
"executable has no parent directory",
)
})?;
if path_dir.file_name() != Some(OsStr::new(PATH_DIRNAME)) {
return Err(io::Error::new(
io::ErrorKind::NotFound,
format!("executable is not inside a {PATH_DIRNAME} directory"),
));
}
let package_dir = path_dir.parent().ok_or_else(|| {
io::Error::new(io::ErrorKind::NotFound, "package has no parent directory")
})?;
if !package_dir.join(PACKAGE_METADATA_FILENAME).is_file() {
return Err(io::Error::new(
io::ErrorKind::NotFound,
format!("package is missing {PACKAGE_METADATA_FILENAME}"),
));
}
let executable_name = current_exe
.file_name()
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "executable has no file name"))?;
let real_rg = package_dir.join(RESOURCES_DIRNAME).join(executable_name);
if !real_rg.is_file() {
return Err(io::Error::new(
io::ErrorKind::NotFound,
format!("bundled ripgrep was not found at {}", real_rg.display()),
));
}
Ok(real_rg)
}
#[cfg(unix)]
fn run_real_rg(real_rg: PathBuf) -> io::Result<i32> {
use std::os::unix::process::CommandExt;
Err(Command::new(real_rg)
.args(std::env::args_os().skip(1))
.exec())
}
#[cfg(windows)]
fn run_real_rg(real_rg: PathBuf) -> io::Result<i32> {
let status = Command::new(real_rg)
.args(std::env::args_os().skip(1))
.status()?;
Ok(status.code().unwrap_or(1))
}
#[cfg(test)]
#[path = "main_tests.rs"]
mod tests;
+37
View File
@@ -0,0 +1,37 @@
use super::PACKAGE_METADATA_FILENAME;
use super::PATH_DIRNAME;
use super::RESOURCES_DIRNAME;
use super::real_rg_path;
use pretty_assertions::assert_eq;
use std::fs;
use tempfile::TempDir;
#[test]
fn resolves_ripgrep_from_package_resources() {
let temp_dir = TempDir::new().expect("temp dir");
let package_dir = temp_dir.path();
let path_dir = package_dir.join(PATH_DIRNAME);
let resources_dir = package_dir.join(RESOURCES_DIRNAME);
fs::create_dir_all(&path_dir).expect("path dir");
fs::create_dir_all(&resources_dir).expect("resources dir");
fs::write(package_dir.join(PACKAGE_METADATA_FILENAME), "{}").expect("metadata");
let executable_name = if cfg!(windows) { "rg.exe" } else { "rg" };
let shim = path_dir.join(executable_name);
let real_rg = resources_dir.join(executable_name);
fs::write(&shim, "shim").expect("shim");
fs::write(&real_rg, "real rg").expect("real rg");
assert_eq!(real_rg_path(&shim).expect("real rg path"), real_rg);
}
#[test]
fn rejects_executable_outside_package_path() {
let temp_dir = TempDir::new().expect("temp dir");
let executable = temp_dir.path().join("rg");
fs::write(&executable, "shim").expect("shim");
let error = real_rg_path(&executable).expect_err("unpackaged shim must fail");
assert_eq!(error.kind(), std::io::ErrorKind::NotFound);
}
+12
View File
@@ -18,6 +18,7 @@ CODEX_RS_ROOT = REPO_ROOT / "codex-rs"
class SourceBuildOutputs:
entrypoint_bin: Path
code_mode_host_bin: Path
rg_shim_bin: Path
bwrap_bin: Path | None
codex_command_runner_bin: Path | None
codex_windows_sandbox_setup_bin: Path | None
@@ -31,6 +32,7 @@ def build_source_binaries(
profile: str,
entrypoint_bin: Path | None,
code_mode_host_bin: Path | None,
rg_shim_bin: Path | None,
bwrap_bin: Path | None,
codex_command_runner_bin: Path | None,
codex_windows_sandbox_setup_bin: Path | None,
@@ -46,6 +48,7 @@ def build_source_binaries(
variant,
build_entrypoint=entrypoint_bin is None,
build_code_mode_host=code_mode_host_bin is None,
build_rg_shim=rg_shim_bin is None,
build_bwrap=spec.is_linux and bwrap_bin is None,
build_codex_command_runner=spec.is_windows and codex_command_runner_bin is None,
build_codex_windows_sandbox_setup=spec.is_windows
@@ -88,6 +91,11 @@ def build_source_binaries(
if code_mode_host_bin is not None
else output_dir / f"codex-code-mode-host{spec.exe_suffix}"
),
rg_shim_bin=(
rg_shim_bin.resolve()
if rg_shim_bin is not None
else output_dir / f"codex-rg{spec.exe_suffix}"
),
bwrap_bin=resolve_output_path(
bwrap_bin,
output_dir / "bwrap" if spec.is_linux else None,
@@ -111,6 +119,7 @@ def source_binaries_for_target(
*,
build_entrypoint: bool,
build_code_mode_host: bool,
build_rg_shim: bool,
build_bwrap: bool,
build_codex_command_runner: bool,
build_codex_windows_sandbox_setup: bool,
@@ -120,6 +129,8 @@ def source_binaries_for_target(
binaries.append(variant.cargo_bin)
if build_code_mode_host:
binaries.append("codex-code-mode-host")
if build_rg_shim:
binaries.append("codex-rg")
if build_bwrap:
binaries.append("bwrap")
if build_codex_command_runner:
@@ -186,6 +197,7 @@ def validate_source_outputs(outputs: SourceBuildOutputs) -> None:
for path in [
outputs.entrypoint_bin,
outputs.code_mode_host_bin,
outputs.rg_shim_bin,
outputs.bwrap_bin,
outputs.codex_command_runner_bin,
outputs.codex_windows_sandbox_setup_bin,
+14
View File
@@ -132,6 +132,14 @@ def parse_args() -> argparse.Namespace:
"scripts/codex_package/rg."
),
)
parser.add_argument(
"--rg-shim-bin",
type=Path,
help=(
"Optional prebuilt Codex ripgrep shim. If omitted, the shim is "
"built with Cargo."
),
)
return parser.parse_args()
@@ -161,6 +169,11 @@ def main() -> int:
"prebuilt code-mode host executable",
"--code-mode-host-bin",
),
rg_shim_bin=resolve_optional_input_path(
args.rg_shim_bin,
"prebuilt Codex ripgrep shim",
"--rg-shim-bin",
),
bwrap_bin=resolve_optional_input_path(
args.bwrap_bin,
"prebuilt Linux bwrap executable",
@@ -181,6 +194,7 @@ def main() -> int:
inputs = PackageInputs(
entrypoint_bin=source_outputs.entrypoint_bin,
code_mode_host_bin=source_outputs.code_mode_host_bin,
rg_shim_bin=source_outputs.rg_shim_bin,
rg_bin=resolve_rg_bin(spec, args.rg_bin),
zsh_bin=resolve_zsh_bin(spec, args.zsh_manifest),
bwrap_bin=source_outputs.bwrap_bin,
+11 -1
View File
@@ -56,7 +56,16 @@ def build_package_dir(
bin_dir / f"codex-code-mode-host{spec.exe_suffix}",
is_windows=spec.is_windows,
)
copy_executable(inputs.rg_bin, path_dir / spec.rg_name, is_windows=spec.is_windows)
copy_executable(
inputs.rg_shim_bin,
path_dir / spec.rg_name,
is_windows=spec.is_windows,
)
copy_executable(
inputs.rg_bin,
resources_dir / spec.rg_name,
is_windows=spec.is_windows,
)
if inputs.zsh_bin is not None:
copy_executable(
@@ -137,6 +146,7 @@ def validate_package_dir(
Path("bin") / variant.entrypoint_name(spec),
Path("bin") / f"codex-code-mode-host{spec.exe_suffix}",
Path("codex-path") / spec.rg_name,
Path("codex-resources") / spec.rg_name,
]
executable_files = list(required_files)
+1
View File
@@ -40,6 +40,7 @@ class PackageVariant:
class PackageInputs:
entrypoint_bin: Path
code_mode_host_bin: Path
rg_shim_bin: Path
rg_bin: Path
zsh_bin: Path | None
bwrap_bin: Path | None
+23
View File
@@ -21,6 +21,7 @@ class SourceBinariesForTargetTest(unittest.TestCase):
PACKAGE_VARIANTS["codex"],
build_entrypoint=False,
build_code_mode_host=False,
build_rg_shim=False,
build_bwrap=False,
build_codex_command_runner=False,
build_codex_windows_sandbox_setup=False,
@@ -37,6 +38,7 @@ class SourceBinariesForTargetTest(unittest.TestCase):
PACKAGE_VARIANTS["codex"],
build_entrypoint=False,
build_code_mode_host=False,
build_rg_shim=False,
build_bwrap=False,
build_codex_command_runner=False,
build_codex_windows_sandbox_setup=False,
@@ -53,6 +55,7 @@ class SourceBinariesForTargetTest(unittest.TestCase):
PACKAGE_VARIANTS["codex"],
build_entrypoint=False,
build_code_mode_host=False,
build_rg_shim=False,
build_bwrap=False,
build_codex_command_runner=False,
build_codex_windows_sandbox_setup=False,
@@ -67,6 +70,7 @@ class SourceBinariesForTargetTest(unittest.TestCase):
PACKAGE_VARIANTS["codex"],
build_entrypoint=False,
build_code_mode_host=False,
build_rg_shim=False,
build_bwrap=False,
build_codex_command_runner=True,
build_codex_windows_sandbox_setup=True,
@@ -81,6 +85,7 @@ class SourceBinariesForTargetTest(unittest.TestCase):
PACKAGE_VARIANTS["codex-app-server"],
build_entrypoint=False,
build_code_mode_host=True,
build_rg_shim=False,
build_bwrap=False,
build_codex_command_runner=False,
build_codex_windows_sandbox_setup=False,
@@ -88,11 +93,27 @@ class SourceBinariesForTargetTest(unittest.TestCase):
["codex-code-mode-host"],
)
def test_missing_rg_shim_is_built(self) -> None:
self.assertEqual(
source_binaries_for_target(
TARGET_SPECS["aarch64-apple-darwin"],
PACKAGE_VARIANTS["codex"],
build_entrypoint=False,
build_code_mode_host=False,
build_rg_shim=True,
build_bwrap=False,
build_codex_command_runner=False,
build_codex_windows_sandbox_setup=False,
),
["codex-rg"],
)
def test_build_uses_prebuilt_windows_helpers_without_running_cargo(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
entrypoint = touch_file(root / "codex.exe")
code_mode_host = touch_file(root / "codex-code-mode-host.exe")
rg_shim = touch_file(root / "codex-rg.exe")
command_runner = touch_file(root / "codex-command-runner.exe")
sandbox_setup = touch_file(root / "codex-windows-sandbox-setup.exe")
@@ -103,6 +124,7 @@ class SourceBinariesForTargetTest(unittest.TestCase):
profile="release",
entrypoint_bin=entrypoint,
code_mode_host_bin=code_mode_host,
rg_shim_bin=rg_shim,
bwrap_bin=None,
codex_command_runner_bin=command_runner,
codex_windows_sandbox_setup_bin=sandbox_setup,
@@ -110,6 +132,7 @@ class SourceBinariesForTargetTest(unittest.TestCase):
self.assertEqual(outputs.entrypoint_bin, entrypoint)
self.assertEqual(outputs.code_mode_host_bin, code_mode_host)
self.assertEqual(outputs.rg_shim_bin, rg_shim)
self.assertEqual(outputs.codex_command_runner_bin, command_runner)
self.assertEqual(outputs.codex_windows_sandbox_setup_bin, sandbox_setup)
+14 -1
View File
@@ -20,10 +20,15 @@ class PackageLayoutTest(unittest.TestCase):
root = Path(temp_dir)
package_dir = root / "package"
package_dir.mkdir()
rg_shim = touch_executable(root / "codex-rg")
rg_shim.write_text("shim", encoding="utf-8")
rg = touch_executable(root / "rg")
rg.write_text("ripgrep", encoding="utf-8")
inputs = PackageInputs(
entrypoint_bin=touch_executable(root / "codex-app-server"),
code_mode_host_bin=touch_executable(root / "codex-code-mode-host"),
rg_bin=touch_executable(root / "rg"),
rg_shim_bin=rg_shim,
rg_bin=rg,
zsh_bin=None,
bwrap_bin=touch_executable(root / "bwrap"),
codex_command_runner_bin=None,
@@ -45,6 +50,14 @@ class PackageLayoutTest(unittest.TestCase):
)
self.assertTrue((package_dir / "bin" / "codex-code-mode-host").is_file())
self.assertEqual(
(package_dir / "codex-path" / "rg").read_text(encoding="utf-8"),
"shim",
)
self.assertEqual(
(package_dir / "codex-resources" / "rg").read_text(encoding="utf-8"),
"ripgrep",
)
def touch_executable(path: Path) -> Path: