Add dynamic HTTP header helpers for MCP servers (#38245)

## What changed

- Add `http_headers_helper` configuration for local streamable HTTP MCP servers. The configured shell command runs once per connection and returns a JSON object of headers that is cached across requests.
- Apply helper headers to MCP startup and OAuth flows while restricting them to the server origin, stopping redirects, rejecting reserved or duplicate headers, and enforcing output and execution limits.
- Reject helpers for remote or managed-disabled servers, use the local environment working directory, and redact helper commands from `codex mcp list` and `codex mcp get` output.

## Testing

- Cover configuration validation, helper lifecycle and output parsing, origin isolation, OAuth discovery and token refresh, managed requirements, environment selection, and CLI redaction.

GitOrigin-RevId: 84e0e26ce75520b0869d37c72b1678e033bd6818
This commit is contained in:
xl-openai authored and copyberry committed 2026-08-12 20:36:32 +00:00
1 parent 8d4d57387a
commit 379cb68444
38 files changed
+1285 -72

No files matched your search

@@ -112,6 +112,17 @@ struct EchoArgs {
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let mut args = std::env::args_os().skip(1);
match args.next().as_deref() {
Some(value) if value == std::ffi::OsStr::new("--http-headers-helper") => {
if std::env::var_os("MCP_TEST_AMBIENT_SECRET").is_some() {
return Err("helper inherited ambient secret".into());
}
println!(r#"{{"Proxy-Authorization":"Bearer gateway-token"}}"#);
return Ok(());
}
_ => {}
}
let bind_addr = parse_bind_addr()?;
let post_failure_state = PostFailureState::default();
const MAX_BIND_RETRIES: u32 = 20;
@@ -209,6 +220,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
} else {
router
};
let router = if let Ok(token) = std::env::var("MCP_EXPECT_GATEWAY_BEARER") {
let expected = Arc::new(format!("Bearer {token}"));
router.layer(middleware::from_fn_with_state(
expected,
require_gateway_bearer,
))
} else {
router
};
axum::serve(listener, router).await?;
task::yield_now().await;
@@ -412,6 +432,25 @@ async fn require_bearer(
}
}
async fn require_gateway_bearer(
State(expected): State<Arc<String>>,
request: Request<Body>,
next: Next,
) -> Result<Response, StatusCode> {
if !request.uri().path().starts_with("/mcp") {
return Ok(next.run(request).await);
}
if request
.headers()
.get("proxy-authorization")
.is_some_and(|value| value.as_bytes() == expected.as_bytes())
{
Ok(next.run(request).await)
} else {
Err(StatusCode::UNAUTHORIZED)
}
}
async fn arm_session_post_failure(
State(state): State<PostFailureState>,
Json(request): Json<ArmSessionPostFailureRequest>,
+366
View File
@@ -0,0 +1,366 @@
use std::collections::HashSet;
use std::fmt;
#[cfg(windows)]
use std::os::windows::process::CommandExt;
use std::path::Path;
use std::path::PathBuf;
use std::process::Stdio;
use std::sync::Arc;
use std::time::Duration;
use anyhow::Result;
use anyhow::anyhow;
use codex_exec_server::ExecServerError;
use codex_exec_server::HttpClient;
use codex_exec_server::HttpHeader;
use codex_exec_server::HttpRedirectPolicy;
use codex_exec_server::HttpRequestParams;
use codex_exec_server::HttpRequestResponse;
use codex_exec_server::HttpResponseBodyStream;
#[cfg(all(unix, not(target_os = "macos")))]
use codex_utils_pty::process_group::kill_process_group;
#[cfg(target_os = "macos")]
use codex_utils_pty::process_group::kill_process_group_with_member_fallback as kill_process_group;
use futures::FutureExt;
use futures::future::BoxFuture;
use futures::future::Shared;
use http::HeaderMap;
use http::HeaderName;
use http::HeaderValue;
use serde::Deserialize;
use serde::de::MapAccess;
use serde::de::Visitor;
use tokio::io::AsyncReadExt;
use tokio::process::Child;
use tokio::process::Command;
use tokio::time::Instant;
use url::Origin;
use url::Url;
use crate::utils::create_env_for_mcp_server;
const HELPER_TIMEOUT: Duration = Duration::from_secs(10);
const MAX_HELPER_OUTPUT_BYTES: usize = 64 * 1024;
type CachedHeaders = Shared<BoxFuture<'static, std::result::Result<Arc<HeaderMap>, Arc<str>>>>;
struct HttpHeadersProvider {
server_origin: Origin,
cached: CachedHeaders,
}
struct HttpHeadersClient {
inner: Arc<dyn HttpClient>,
provider: HttpHeadersProvider,
}
struct HelperProcess {
child: Child,
#[cfg(unix)]
process_group_id: u32,
#[cfg(windows)]
job: codex_utils_pty::JobObject,
}
struct RawHeaderEntries {
// Keep raw entries because ordinary map deserialization collapses exact duplicate keys.
entries: Vec<(String, String)>,
has_exact_duplicate: bool,
}
struct RawHeaderEntriesVisitor;
impl<'de> Visitor<'de> for RawHeaderEntriesVisitor {
type Value = RawHeaderEntries;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("a JSON object of string header names and values")
}
fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
where
A: MapAccess<'de>,
{
let mut entries = Vec::with_capacity(map.size_hint().unwrap_or_default());
let mut names = HashSet::new();
let mut has_exact_duplicate = false;
while let Some((name, value)) = map.next_entry::<String, String>()? {
has_exact_duplicate |= !names.insert(name.clone());
entries.push((name, value));
}
Ok(RawHeaderEntries {
entries,
has_exact_duplicate,
})
}
}
impl<'de> Deserialize<'de> for RawHeaderEntries {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
deserializer.deserialize_map(RawHeaderEntriesVisitor)
}
}
impl Drop for HelperProcess {
fn drop(&mut self) {
#[cfg(unix)]
let _ = kill_process_group(self.process_group_id);
#[cfg(windows)]
let _ = self.job.terminate();
let _ = self.child.start_kill();
}
}
impl HttpHeadersProvider {
fn new(server_url: &str, command: &str, cwd: PathBuf) -> Result<Self> {
let command = command.to_string();
let cached = async move {
run_helper(&command, &cwd)
.await
.map(Arc::new)
.map_err(|error| Arc::<str>::from(error.to_string()))
}
.boxed()
.shared();
Ok(Self {
server_origin: Url::parse(server_url)?.origin(),
cached,
})
}
async fn headers(&self) -> Result<Arc<HeaderMap>, ExecServerError> {
self.cached
.clone()
.await
.map_err(|error| ExecServerError::HttpRequest(error.to_string()))
}
}
/// No rejection-driven refresh: 401/403 may be OAuth challenges, and reconnecting loses sessions.
pub fn with_http_headers_helper(
inner: Arc<dyn HttpClient>,
server_url: &str,
command: &str,
cwd: PathBuf,
) -> Result<Arc<dyn HttpClient>> {
let provider = HttpHeadersProvider::new(server_url, command, cwd)?;
Ok(Arc::new(HttpHeadersClient { inner, provider }))
}
impl HttpHeadersClient {
async fn prepare_request(
&self,
mut params: HttpRequestParams,
) -> Result<HttpRequestParams, ExecServerError> {
let Ok(url) = Url::parse(&params.url) else {
return Ok(params);
};
if self.provider.server_origin != url.origin() {
return Ok(params);
}
// TODO: Follow same-origin redirects once later hops cannot leak helper headers.
params.redirect_policy = HttpRedirectPolicy::Stop;
let deadline = params
.timeout_ms
.map(|timeout_ms| Instant::now() + Duration::from_millis(timeout_ms));
let headers = match deadline {
Some(deadline) => tokio::time::timeout_at(deadline, self.provider.headers())
.await
.map_err(|_| {
ExecServerError::HttpRequest("HTTP request timed out".to_string())
})??,
None => self.provider.headers().await?,
};
for (name, value) in headers.iter() {
params
.headers
.retain(|header| !header.name.eq_ignore_ascii_case(name.as_str()));
params.headers.push(HttpHeader {
name: name.to_string(),
value: std::str::from_utf8(value.as_bytes())
.map_err(|error| ExecServerError::HttpRequest(error.to_string()))?
.to_string(),
});
}
if let Some(deadline) = deadline {
let remaining = deadline.saturating_duration_since(Instant::now());
params.timeout_ms = Some(
u64::try_from(remaining.as_millis())
.unwrap_or(u64::MAX)
.max(1),
);
}
Ok(params)
}
}
impl HttpClient for HttpHeadersClient {
fn http_request(
&self,
params: HttpRequestParams,
) -> BoxFuture<'_, Result<HttpRequestResponse, ExecServerError>> {
async move {
let params = self.prepare_request(params).await?;
self.inner.http_request(params).await
}
.boxed()
}
fn http_request_stream(
&self,
params: HttpRequestParams,
) -> BoxFuture<'_, Result<(HttpRequestResponse, HttpResponseBodyStream), ExecServerError>> {
async move {
let params = self.prepare_request(params).await?;
self.inner.http_request_stream(params).await
}
.boxed()
}
}
async fn run_helper(command: &str, cwd: &Path) -> Result<HeaderMap> {
#[cfg(windows)]
let shell = std::env::var_os("COMSPEC").unwrap_or_else(|| "cmd.exe".into());
#[cfg(not(windows))]
let shell = "sh";
// Match the repository's existing shell-command convention. The command is ordinary
// configuration and may be visible in local process metadata; credentials belong in the
// JSON output rather than in the command text.
let mut process = Command::new(shell);
#[cfg(windows)]
{
process.args(["/Q", "/D", "/C"]);
process.as_std_mut().raw_arg(format!(r#""{command}""#));
}
#[cfg(not(windows))]
process.args(["-c", command]);
#[cfg(unix)]
process.process_group(0);
process
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.current_dir(cwd)
// Match local MCP subprocess policy; arbitrary ambient variables are not inherited.
.env_clear()
.envs(create_env_for_mcp_server(/*extra_env*/ None, &[])?)
.kill_on_drop(true);
#[cfg(windows)]
let (child, job) = {
let job = codex_utils_pty::JobObject::create_without_breakaway()
.map_err(|error| anyhow!("MCP HTTP headers helper containment failed: {error}"))?;
let child = job
.spawn_contained(&mut process)
.map_err(|error| anyhow!("MCP HTTP headers helper failed to start: {error}"))?;
(child, job)
};
#[cfg(not(windows))]
let child = process
.spawn()
.map_err(|error| anyhow!("MCP HTTP headers helper failed to start: {error}"))?;
let mut process = HelperProcess {
#[cfg(unix)]
process_group_id: child
.id()
.ok_or_else(|| anyhow!("MCP HTTP headers helper process id was unavailable"))?,
child,
#[cfg(windows)]
job,
};
let output = tokio::time::timeout(HELPER_TIMEOUT, async {
let stdout = process
.child
.stdout
.take()
.ok_or_else(|| anyhow!("MCP HTTP headers helper stdout was unavailable"))?;
let mut output = Vec::new();
stdout
.take((MAX_HELPER_OUTPUT_BYTES + 1) as u64)
.read_to_end(&mut output)
.await?;
if output.len() > MAX_HELPER_OUTPUT_BYTES {
return Err(anyhow!("MCP HTTP headers helper output exceeds 64 KiB"));
}
let status = process.child.wait().await?;
if !status.success() {
return Err(anyhow!(
"MCP HTTP headers helper exited with status {status}"
));
}
Ok(output)
})
.await
.map_err(|_| anyhow!("MCP HTTP headers helper timed out after 10 seconds"))??;
parse_helper_output(output)
}
fn parse_helper_output(stdout: Vec<u8>) -> Result<HeaderMap> {
let stdout = String::from_utf8(stdout)
.map_err(|_| anyhow!("MCP HTTP headers helper wrote non-UTF-8 data"))?;
let mut deserializer = serde_json::Deserializer::from_str(stdout.trim());
let headers = RawHeaderEntries::deserialize(&mut deserializer)
.and_then(|headers| {
deserializer.end()?;
Ok(headers)
})
.map_err(|_| anyhow!("MCP HTTP headers helper must output a JSON object of strings"))?;
if headers.has_exact_duplicate {
return Err(anyhow!(
"MCP HTTP headers helper returned duplicate header names"
));
}
let mut parsed = HeaderMap::with_capacity(headers.entries.len());
for (name, value) in headers.entries {
let name = HeaderName::from_bytes(name.as_bytes())
.map_err(|_| anyhow!("MCP HTTP headers helper returned an invalid header name"))?;
// Helper values replace same-name configured headers; bearer/OAuth owns Authorization.
// Google IAP uses Proxy-Authorization alongside application Authorization. For HTTPS MCP
// URLs it is sent through the forward-proxy tunnel to IAP, not used as CONNECT auth.
if matches!(
name.as_str(),
"accept"
| "authorization"
| "connection"
| "content-encoding"
| "content-length"
| "content-type"
| "host"
| "keep-alive"
| "last-event-id"
| "mcp-protocol-version"
| "mcp-session-id"
| "origin"
| "proxy-connection"
| "referer"
| "te"
| "trailer"
| "transfer-encoding"
| "upgrade"
) {
return Err(anyhow!(
"MCP HTTP headers helper returned a reserved header"
));
}
if parsed.contains_key(&name) {
return Err(anyhow!(
"MCP HTTP headers helper returned duplicate header names"
));
}
let value = HeaderValue::from_str(&value)
.map_err(|_| anyhow!("MCP HTTP headers helper returned an invalid header value"))?;
parsed.insert(name, value);
}
Ok(parsed)
}
#[cfg(test)]
#[path = "http_headers_tests.rs"]
mod tests;
@@ -0,0 +1,200 @@
use super::*;
use pretty_assertions::assert_eq;
#[test]
fn helper_output_errors_do_not_echo_secrets() {
for output in [
br#"{"Authorization":"secret"}"#.as_slice(),
br#"{"secret":"secret","secret":"secret"}"#.as_slice(),
br#"{"secret":"secret","Secret":"secret"}"#.as_slice(),
] {
let error = parse_helper_output(output.to_vec()).expect_err("invalid helper output");
assert!(!error.to_string().contains("secret"));
}
}
#[cfg(unix)]
#[tokio::test]
async fn helper_attempt_is_shared_after_cancellation() {
use tempfile::tempdir;
let temp = tempdir().expect("temporary helper directory");
let cwd = temp
.path()
.canonicalize()
.expect("canonical helper directory");
let cancelled_invocations = cwd.join("cancelled-invocations");
let cancelled = HttpHeadersProvider::new(
"https://example.com",
&format!(
"test \"$(pwd)\" = '{0}'; test -n \"$HOME\"; test -n \"$PATH\"; \
printf x >> '{1}'; sleep 0.2; printf '{{\"X-Gateway\":\"token\"}}'",
cwd.display(),
cancelled_invocations.display(),
),
cwd.clone(),
)
.expect("cancelled provider");
assert!(
tokio::time::timeout(Duration::from_millis(20), cancelled.headers())
.await
.is_err()
);
assert!(cancelled.headers().await.is_ok());
assert_eq!(
std::fs::read_to_string(cancelled_invocations).expect("cancelled invocation count"),
"x"
);
let dropped_started = cwd.join("dropped-helper-started");
let dropped_finished = cwd.join("dropped-helper-finished");
let dropped = HttpHeadersProvider::new(
"https://example.com",
&format!(
"printf x > '{}'; sleep 1; printf x > '{}'",
dropped_started.display(),
dropped_finished.display(),
),
cwd,
)
.expect("dropped provider");
assert!(
tokio::time::timeout(Duration::from_millis(500), dropped.headers())
.await
.is_err()
);
assert!(dropped_started.exists());
drop(dropped);
tokio::time::sleep(Duration::from_millis(1_100)).await;
assert!(!dropped_finished.exists());
}
#[tokio::test]
async fn nonzero_helper_exit_is_cached() {
let temp = tempfile::tempdir().expect("temporary helper directory");
let failed_invocations = temp.path().join("failed-invocations");
let command = if cfg!(windows) {
format!(
r#"echo x>>"{0}" & echo {{"X-Gateway":"valid"}} & exit /b 23"#,
failed_invocations.display()
)
} else {
format!(
"echo x >> '{0}'; printf '{{\"X-Gateway\":\"valid\"}}'; exit 23",
failed_invocations.display()
)
};
let failed = HttpHeadersProvider::new(
"https://example.com/mcp",
&command,
temp.path().to_path_buf(),
)
.expect("failed provider");
let first = failed.headers().await.expect_err("failed helper");
let second = failed.headers().await.expect_err("cached failure");
assert_eq!(first.to_string(), second.to_string());
let invocations = std::fs::read_to_string(failed_invocations).expect("failed invocation count");
assert_eq!(invocations.lines().count(), 1);
}
#[cfg(unix)]
#[tokio::test]
async fn connection_headers_are_cached_and_origin_bound() {
use axum::Router;
use axum::http::StatusCode;
use axum::response::Redirect;
use axum::routing::get;
use axum::routing::post;
use codex_exec_server::RouteAwareHttpClient;
use codex_http_client::HttpClientFactory;
use codex_http_client::OutboundProxyPolicy;
use std::sync::Arc;
use tempfile::tempdir;
use tokio::net::TcpListener;
async fn handle(headers: axum::http::HeaderMap) -> StatusCode {
assert_eq!(
headers.get("proxy-authorization"),
Some(&HeaderValue::from_static("Bearer token"))
);
assert_eq!(
headers.get("x-label"),
Some(&HeaderValue::from_bytes("café".as_bytes()).unwrap())
);
StatusCode::NO_CONTENT
}
let temp = tempdir().expect("temporary helper directory");
let invocation_file = temp.path().join("invocations");
let cross_listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind cross-origin server");
let cross_url = format!("http://{}/start", cross_listener.local_addr().unwrap());
tokio::spawn(async move {
axum::serve(
cross_listener,
Router::new()
.route("/start", get(|| async { Redirect::temporary("/final") }))
.route("/final", get(|| async { StatusCode::NO_CONTENT })),
)
.await
.expect("serve cross-origin requests");
});
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind test server");
let url = format!("http://{}/mcp", listener.local_addr().unwrap());
let redirect_url = cross_url.clone();
let app = Router::new().route("/mcp", post(handle)).route(
"/redirect",
get(move || std::future::ready(Redirect::temporary(&redirect_url))),
);
tokio::spawn(async move {
axum::serve(listener, app)
.await
.expect("serve test requests");
});
let command = format!(
"printf x >> '{}'; printf '{{\"Proxy-Authorization\":\"Bearer token\",\"X-Label\":\"café\"}}'",
invocation_file.display(),
);
let inner: Arc<dyn HttpClient> = Arc::new(RouteAwareHttpClient::new(HttpClientFactory::new(
OutboundProxyPolicy::ReqwestDefault,
)));
let client = with_http_headers_helper(inner, &url, &command, temp.path().to_path_buf())
.expect("headers helper client");
let request = |session: &str| HttpRequestParams {
method: "POST".to_string(),
url: url.clone(),
headers: Vec::new(),
body: None,
timeout_ms: Some(5_000),
redirect_policy: HttpRedirectPolicy::Follow,
request_id: session.to_string(),
stream_response: true,
};
let mut cross_request = request("cross-origin");
cross_request.method = "GET".to_string();
cross_request.url = cross_url;
assert_eq!(
client.http_request(cross_request).await.unwrap().status,
204
);
assert!(!invocation_file.exists());
let mut redirect_request = request("redirect");
redirect_request.method = "GET".to_string();
redirect_request.url = url.replace("/mcp", "/redirect");
assert_eq!(
client.http_request(redirect_request).await.unwrap().status,
307
);
let (left, right) = tokio::join!(
client.http_request_stream(request("session-a")),
client.http_request_stream(request("session-b"))
);
assert_eq!(left.expect("left request").0.status, 204);
assert_eq!(right.expect("right request").0.status, 204);
assert_eq!(
std::fs::read_to_string(&invocation_file).expect("helper invocation count"),
"x"
);
}
+2
View File
@@ -3,6 +3,7 @@ mod elicitation_client_service;
mod event_notification_transport;
mod executor_process_transport;
mod http_client_adapter;
mod http_headers;
mod in_process_transport;
mod incoming_jsonrpc;
mod local_stdio_transport;
@@ -28,6 +29,7 @@ pub use auth_status::discover_streamable_http_oauth;
pub use codex_protocol::protocol::McpAuthStatus;
pub use event_notification_transport::EventNotificationReceiver;
pub use http_client_adapter::StreamableHttpRedirectMode;
pub use http_headers::with_http_headers_helper;
pub use in_process_transport::InProcessTransportFactory;
pub use oauth::StoredOAuthCredentialSnapshot;
pub use oauth::StoredOAuthTokens;