Files
codex/codex-rs/hooks/src/engine/command_runner.rs
T
Abhinav dd916428cd Terminate timed-out hook process trees (#37527)
## Why

Timing out a command hook can leave descendant processes running after the hook itself is stopped.

## What changed

- Run hook commands in a process group on Unix and a job object on Windows.
- Terminate the full process tree when hook execution is cancelled or fails, with `taskkill` as a Windows fallback.
- Preserve descendants when a hook completes successfully so intentionally detached helpers can continue running.

GitOrigin-RevId: 1a18653d34c6f1685ef8c4f8e8546663b276292a
2026-08-08 03:48:34 +00:00

307 lines
8.8 KiB
Rust

use std::io::ErrorKind;
use std::path::Path;
use std::process::Stdio;
use std::time::Duration;
use std::time::Instant;
#[cfg(windows)]
use codex_utils_pty::JobObject;
use tokio::io::AsyncWriteExt;
use tokio::process::Command;
use tokio::time::timeout;
use tracing::Span;
use super::CommandShell;
use super::ConfiguredHandler;
use super::dispatcher::hook_event_name_label;
use super::dispatcher::hook_execution_mode_label;
use super::dispatcher::hook_handler_type_label;
use super::dispatcher::hook_scope_label;
use super::dispatcher::hook_source_label;
use super::dispatcher::scope_for_event;
use crate::output_spill::HookOutputSpiller;
use codex_protocol::protocol::HookExecutionMode;
use codex_protocol::protocol::HookHandlerType;
/// Owns the shell and output spiller shared by a session's command hooks.
#[derive(Clone)]
pub(crate) struct CommandHookRuntime {
shell: CommandShell,
output_spiller: HookOutputSpiller,
}
impl CommandHookRuntime {
pub(crate) fn new(shell: CommandShell) -> Self {
Self {
shell,
output_spiller: HookOutputSpiller::new(),
}
}
pub(crate) fn output_spiller(&self) -> &HookOutputSpiller {
&self.output_spiller
}
}
#[derive(Debug)]
pub(crate) struct CommandRunResult {
pub started_at: i64,
pub completed_at: i64,
pub duration_ms: i64,
pub exit_code: Option<i32>,
pub stdout: String,
pub stderr: String,
pub error: Option<String>,
}
#[tracing::instrument(
name = "codex.hooks.command",
level = "trace",
skip_all,
fields(
hook.event_name = hook_event_name_label(handler.event_name),
hook.handler_type = hook_handler_type_label(HookHandlerType::Command),
hook.execution_mode = hook_execution_mode_label(HookExecutionMode::Sync),
hook.scope = hook_scope_label(scope_for_event(handler.event_name)),
hook.source = hook_source_label(handler.source),
hook.display_order = handler.display_order,
hook.configured_order = configured_order,
hook.timeout_sec = handler.timeout_sec,
hook.command_outcome = tracing::field::Empty,
)
)]
pub(crate) async fn run_command(
runtime: &CommandHookRuntime,
handler: &ConfiguredHandler,
configured_order: usize,
input_json: &str,
cwd: &Path,
) -> CommandRunResult {
let started_at = chrono::Utc::now().timestamp();
let started = Instant::now();
let mut command = build_command(&runtime.shell, handler);
command
.current_dir(cwd)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true);
#[cfg(unix)]
command.process_group(0);
#[cfg(windows)]
let mut process_tree_job = JobObject::create().ok();
#[cfg(windows)]
let child = match process_tree_job.as_ref() {
Some(job) => match job.spawn_contained(&mut command) {
Ok(child) => Ok(child),
Err(_) => {
process_tree_job = None;
command.creation_flags(0);
command.spawn()
}
},
None => command.spawn(),
};
#[cfg(not(windows))]
let child = command.spawn();
let mut child = match child {
Ok(child) => child,
Err(err) => {
return finish_command_run(
started_at,
started,
CommandRunCompletion {
exit_code: None,
stdout: String::new(),
stderr: String::new(),
error: Some(err.to_string()),
outcome: "spawn_error",
},
);
}
};
let mut process_tree_guard = ProcessTreeGuard {
process_id: child.id(),
#[cfg(windows)]
job: process_tree_job,
};
if let Some(mut stdin) = child.stdin.take()
&& let Err(err) = stdin.write_all(input_json.as_bytes()).await
&& err.kind() != ErrorKind::BrokenPipe
{
let _ = child.kill().await;
return finish_command_run(
started_at,
started,
CommandRunCompletion {
exit_code: None,
stdout: String::new(),
stderr: String::new(),
error: Some(format!("failed to write hook stdin: {err}")),
outcome: "stdin_error",
},
);
}
let timeout_duration = Duration::from_secs(handler.timeout_sec);
match timeout(timeout_duration, child.wait_with_output()).await {
Ok(Ok(output)) => {
// Successfully completed hooks may intentionally leave detached helpers running.
#[cfg(windows)]
if let Some(job) = process_tree_guard.job.as_ref() {
let _ = job.preserve_descendants();
}
process_tree_guard.process_id = None;
finish_command_run(
started_at,
started,
CommandRunCompletion {
exit_code: output.status.code(),
stdout: String::from_utf8_lossy(&output.stdout).to_string(),
stderr: String::from_utf8_lossy(&output.stderr).to_string(),
error: None,
outcome: "completed",
},
)
}
Ok(Err(err)) => finish_command_run(
started_at,
started,
CommandRunCompletion {
exit_code: None,
stdout: String::new(),
stderr: String::new(),
error: Some(err.to_string()),
outcome: "wait_error",
},
),
Err(_) => finish_command_run(
started_at,
started,
CommandRunCompletion {
exit_code: None,
stdout: String::new(),
stderr: String::new(),
error: Some(format!("hook timed out after {}s", handler.timeout_sec)),
outcome: "timeout",
},
),
}
}
// Needed only until command hooks move to the exec server, which owns process-tree cleanup.
struct ProcessTreeGuard {
process_id: Option<u32>,
#[cfg(windows)]
job: Option<JobObject>,
}
impl Drop for ProcessTreeGuard {
fn drop(&mut self) {
let Some(process_id) = self.process_id else {
return;
};
#[cfg(unix)]
{
let _ = codex_utils_pty::process_group::kill_process_group(process_id);
}
#[cfg(windows)]
{
if let Some(job) = self.job.as_ref() {
let _ = job.terminate();
} else {
let _ = std::process::Command::new("taskkill")
.args(["/PID", &process_id.to_string(), "/T", "/F"])
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn();
}
}
}
}
struct CommandRunCompletion {
exit_code: Option<i32>,
stdout: String,
stderr: String,
error: Option<String>,
outcome: &'static str,
}
fn finish_command_run(
started_at: i64,
started: Instant,
completion: CommandRunCompletion,
) -> CommandRunResult {
Span::current().record("hook.command_outcome", completion.outcome);
CommandRunResult {
started_at,
completed_at: chrono::Utc::now().timestamp(),
duration_ms: started.elapsed().as_millis().try_into().unwrap_or(i64::MAX),
exit_code: completion.exit_code,
stdout: completion.stdout,
stderr: completion.stderr,
error: completion.error,
}
}
fn build_command(shell: &CommandShell, handler: &ConfiguredHandler) -> Command {
let mut command = if shell.program.is_empty() {
default_shell_command()
} else {
Command::new(&shell.program)
};
if shell.program.is_empty() {
#[cfg(windows)]
command.raw_arg(format!(r#""{}""#, handler.command));
#[cfg(not(windows))]
command.arg(&handler.command);
} else {
command.args(&shell.args);
#[cfg(windows)]
if shell.args.iter().any(|arg| arg.eq_ignore_ascii_case("/c")) {
command.raw_arg(format!(r#""{}""#, handler.command));
} else {
command.arg(&handler.command);
}
#[cfg(not(windows))]
command.arg(&handler.command);
}
command.envs(&handler.env);
command
}
fn default_shell_command() -> Command {
#[cfg(windows)]
{
let comspec = std::env::var("COMSPEC").unwrap_or_else(|_| "cmd.exe".to_string());
let mut command = Command::new(comspec);
command.arg("/C");
command
}
#[cfg(not(windows))]
{
let shell = std::env::var("SHELL").unwrap_or_else(|_| "/bin/sh".to_string());
let mut command = Command::new(shell);
command.arg("-lc");
command
}
}
#[cfg(test)]
#[path = "command_runner_tests.rs"]
mod tests;