[codex][otel] support mtls configuration (#6228)

fix for https://github.com/openai/codex/issues/6153

supports mTLS configuration and includes TLS features in the library
build to enable secure HTTPS connections with custom root certificates.

grpc:
https://docs.rs/tonic/0.13.1/src/tonic/transport/channel/endpoint.rs.html#63
https:
https://docs.rs/reqwest/0.12.23/src/reqwest/async_impl/client.rs.html#516

codex-rs/Cargo.lock

@@ -1366,6 +1366,7 @@ dependencies = [
  "codex-app-server-protocol",
  "codex-protocol",
  "eventsource-stream",
+ "http",
  "opentelemetry",
  "opentelemetry-otlp",
  "opentelemetry-semantic-conventions",
@@ -5174,6 +5175,7 @@ version = "0.23.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2491382039b29b9b11ff08b76ff6c97cf287671dbb74f0be44bda389fffe9bd1"
 dependencies = [
+ "log",
  "once_cell",
  "ring",
  "rustls-pki-types",
@@ -6601,8 +6603,10 @@ dependencies = [
  "percent-encoding",
  "pin-project",
  "prost",
+ "rustls-native-certs",
  "socket2 0.5.10",
  "tokio",
+ "tokio-rustls",
  "tokio-stream",
  "tower",
  "tower-layer",

codex-rs/core/src/config/types.rs

@@ -282,6 +282,14 @@ pub enum OtelHttpProtocol {
     Json,
 }
 
+#[derive(Deserialize, Debug, Clone, PartialEq, Default)]
+#[serde(rename_all = "kebab-case")]
+pub struct OtelTlsConfig {
+    pub ca_certificate: Option<PathBuf>,
+    pub client_certificate: Option<PathBuf>,
+    pub client_private_key: Option<PathBuf>,
+}
+
 /// Which OTEL exporter to use.
 #[derive(Deserialize, Debug, Clone, PartialEq)]
 #[serde(rename_all = "kebab-case")]
@@ -289,12 +297,18 @@ pub enum OtelExporterKind {
     None,
     OtlpHttp {
         endpoint: String,
+        #[serde(default)]
         headers: HashMap<String, String>,
         protocol: OtelHttpProtocol,
+        #[serde(default)]
+        tls: Option<OtelTlsConfig>,
     },
     OtlpGrpc {
         endpoint: String,
+        #[serde(default)]
         headers: HashMap<String, String>,
+        #[serde(default)]
+        tls: Option<OtelTlsConfig>,
     },
 }
 

codex-rs/core/src/otel_init.rs

@@ -5,6 +5,7 @@ use crate::default_client::originator;
 use codex_otel::config::OtelExporter;
 use codex_otel::config::OtelHttpProtocol;
 use codex_otel::config::OtelSettings;
+use codex_otel::config::OtelTlsConfig as OtelTlsSettings;
 use codex_otel::otel_provider::OtelProvider;
 use std::error::Error;
 
@@ -21,6 +22,7 @@ pub fn build_provider(
             endpoint,
             headers,
             protocol,
+            tls,
         } => {
             let protocol = match protocol {
                 Protocol::Json => OtelHttpProtocol::Json,
@@ -34,14 +36,28 @@ pub fn build_provider(
                     .map(|(k, v)| (k.clone(), v.clone()))
                     .collect(),
                 protocol,
+                tls: tls.as_ref().map(|config| OtelTlsSettings {
+                    ca_certificate: config.ca_certificate.clone(),
+                    client_certificate: config.client_certificate.clone(),
+                    client_private_key: config.client_private_key.clone(),
+                }),
             }
         }
-        Kind::OtlpGrpc { endpoint, headers } => OtelExporter::OtlpGrpc {
+        Kind::OtlpGrpc {
+            endpoint,
+            headers,
+            tls,
+        } => OtelExporter::OtlpGrpc {
             endpoint: endpoint.clone(),
             headers: headers
                 .iter()
                 .map(|(k, v)| (k.clone(), v.clone()))
                 .collect(),
+            tls: tls.as_ref().map(|config| OtelTlsSettings {
+                ca_certificate: config.ca_certificate.clone(),
+                client_certificate: config.client_certificate.clone(),
+                client_private_key: config.client_private_key.clone(),
+            }),
         },
     };
 

codex-rs/otel/Cargo.toml

@@ -27,18 +27,26 @@ opentelemetry-otlp = { workspace = true, features = [
     "grpc-tonic",
     "http-proto",
     "http-json",
+    "logs",
     "reqwest",
     "reqwest-rustls",
+    "tls",
+    "tls-roots",
 ], optional = true }
 opentelemetry-semantic-conventions = { workspace = true }
 opentelemetry_sdk = { workspace = true, features = [
     "logs",
     "rt-tokio",
 ], optional = true }
+http = { workspace = true }
 reqwest = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 strum_macros = { workspace = true }
 tokio = { workspace = true }
-tonic = { workspace = true, optional = true }
+tonic = { workspace = true, optional = true, features = [
+    "transport",
+    "tls-native-roots",
+    "tls-ring",
+] }
 tracing = { workspace = true }

codex-rs/otel/src/config.rs

@@ -18,16 +18,25 @@ pub enum OtelHttpProtocol {
     Json,
 }
 
+#[derive(Clone, Debug, Default)]
+pub struct OtelTlsConfig {
+    pub ca_certificate: Option<PathBuf>,
+    pub client_certificate: Option<PathBuf>,
+    pub client_private_key: Option<PathBuf>,
+}
+
 #[derive(Clone, Debug)]
 pub enum OtelExporter {
     None,
     OtlpGrpc {
         endpoint: String,
         headers: HashMap<String, String>,
+        tls: Option<OtelTlsConfig>,
     },
     OtlpHttp {
         endpoint: String,
         headers: HashMap<String, String>,
         protocol: OtelHttpProtocol,
+        tls: Option<OtelTlsConfig>,
     },
 }

codex-rs/otel/src/otel_provider.rs

@@ -1,8 +1,13 @@
 use crate::config::OtelExporter;
 use crate::config::OtelHttpProtocol;
 use crate::config::OtelSettings;
+use crate::config::OtelTlsConfig;
+use http::Uri;
 use opentelemetry::KeyValue;
 use opentelemetry_otlp::LogExporter;
+use opentelemetry_otlp::OTEL_EXPORTER_OTLP_LOGS_TIMEOUT;
+use opentelemetry_otlp::OTEL_EXPORTER_OTLP_TIMEOUT;
+use opentelemetry_otlp::OTEL_EXPORTER_OTLP_TIMEOUT_DEFAULT;
 use opentelemetry_otlp::Protocol;
 use opentelemetry_otlp::WithExportConfig;
 use opentelemetry_otlp::WithHttpConfig;
@@ -10,11 +15,23 @@ use opentelemetry_otlp::WithTonicConfig;
 use opentelemetry_sdk::Resource;
 use opentelemetry_sdk::logs::SdkLoggerProvider;
 use opentelemetry_semantic_conventions as semconv;
+use reqwest::Certificate as ReqwestCertificate;
+use reqwest::Identity as ReqwestIdentity;
 use reqwest::header::HeaderMap;
 use reqwest::header::HeaderName;
 use reqwest::header::HeaderValue;
+use std::env;
 use std::error::Error;
+use std::fs;
+use std::io::ErrorKind;
+use std::io::{self};
+use std::path::Path;
+use std::path::PathBuf;
+use std::time::Duration;
 use tonic::metadata::MetadataMap;
+use tonic::transport::Certificate as TonicCertificate;
+use tonic::transport::ClientTlsConfig;
+use tonic::transport::Identity as TonicIdentity;
 use tracing::debug;
 
 const ENV_ATTRIBUTE: &str = "env";
@@ -47,8 +64,12 @@ impl OtelProvider {
                 debug!("No exporter enabled in OTLP settings.");
                 return Ok(None);
             }
-            OtelExporter::OtlpGrpc { endpoint, headers } => {
+            OtelExporter::OtlpGrpc {
-                debug!("Using OTLP Grpc exporter: {}", endpoint);
+                endpoint,
+                headers,
+                tls,
+            } => {
+                debug!("Using OTLP Grpc exporter: {endpoint}");
 
                 let mut header_map = HeaderMap::new();
                 for (key, value) in headers {
@@ -59,10 +80,25 @@ impl OtelProvider {
                     }
                 }
 
+                let base_tls_config = ClientTlsConfig::new()
+                    .with_enabled_roots()
+                    .assume_http2(true);
+
+                let tls_config = match tls.as_ref() {
+                    Some(tls) => build_grpc_tls_config(
+                        endpoint,
+                        base_tls_config,
+                        tls,
+                        settings.codex_home.as_path(),
+                    )?,
+                    None => base_tls_config,
+                };
+
                 let exporter = LogExporter::builder()
                     .with_tonic()
                     .with_endpoint(endpoint)
                     .with_metadata(MetadataMap::from_headers(header_map))
+                    .with_tls_config(tls_config)
                     .build()?;
 
                 builder = builder.with_batch_exporter(exporter);
@@ -71,20 +107,27 @@ impl OtelProvider {
                 endpoint,
                 headers,
                 protocol,
+                tls,
             } => {
-                debug!("Using OTLP Http exporter: {}", endpoint);
+                debug!("Using OTLP Http exporter: {endpoint}");
 
                 let protocol = match protocol {
                     OtelHttpProtocol::Binary => Protocol::HttpBinary,
                     OtelHttpProtocol::Json => Protocol::HttpJson,
                 };
 
-                let exporter = LogExporter::builder()
+                let mut exporter_builder = LogExporter::builder()
                     .with_http()
                     .with_endpoint(endpoint)
                     .with_protocol(protocol)
-                    .with_headers(headers.clone())
+                    .with_headers(headers.clone());
-                    .build()?;
+
+                if let Some(tls) = tls.as_ref() {
+                    let client = build_http_client(tls, settings.codex_home.as_path())?;
+                    exporter_builder = exporter_builder.with_http_client(client);
+                }
+
+                let exporter = exporter_builder.build()?;
 
                 builder = builder.with_batch_exporter(exporter);
             }
@@ -101,3 +144,127 @@ impl Drop for OtelProvider {
         let _ = self.logger.shutdown();
     }
 }
+
+fn build_grpc_tls_config(
+    endpoint: &str,
+    tls_config: ClientTlsConfig,
+    tls: &OtelTlsConfig,
+    codex_home: &Path,
+) -> Result<ClientTlsConfig, Box<dyn Error>> {
+    let uri: Uri = endpoint.parse()?;
+    let host = uri.host().ok_or_else(|| {
+        config_error(format!(
+            "OTLP gRPC endpoint {endpoint} does not include a host"
+        ))
+    })?;
+
+    let mut config = tls_config.domain_name(host.to_owned());
+
+    if let Some(path) = tls.ca_certificate.as_ref() {
+        let (pem, _) = read_bytes(codex_home, path)?;
+        config = config.ca_certificate(TonicCertificate::from_pem(pem));
+    }
+
+    match (&tls.client_certificate, &tls.client_private_key) {
+        (Some(cert_path), Some(key_path)) => {
+            let (cert_pem, _) = read_bytes(codex_home, cert_path)?;
+            let (key_pem, _) = read_bytes(codex_home, key_path)?;
+            config = config.identity(TonicIdentity::from_pem(cert_pem, key_pem));
+        }
+        (Some(_), None) | (None, Some(_)) => {
+            return Err(config_error(
+                "client_certificate and client_private_key must both be provided for mTLS",
+            ));
+        }
+        (None, None) => {}
+    }
+
+    Ok(config)
+}
+
+fn build_http_client(
+    tls: &OtelTlsConfig,
+    codex_home: &Path,
+) -> Result<reqwest::Client, Box<dyn Error>> {
+    let mut builder =
+        reqwest::Client::builder().timeout(resolve_otlp_timeout(OTEL_EXPORTER_OTLP_LOGS_TIMEOUT));
+
+    if let Some(path) = tls.ca_certificate.as_ref() {
+        let (pem, location) = read_bytes(codex_home, path)?;
+        let certificate = ReqwestCertificate::from_pem(pem.as_slice()).map_err(|error| {
+            config_error(format!(
+                "failed to parse certificate {}: {error}",
+                location.display()
+            ))
+        })?;
+        builder = builder.add_root_certificate(certificate);
+    }
+
+    match (&tls.client_certificate, &tls.client_private_key) {
+        (Some(cert_path), Some(key_path)) => {
+            let (mut cert_pem, cert_location) = read_bytes(codex_home, cert_path)?;
+            let (key_pem, key_location) = read_bytes(codex_home, key_path)?;
+            cert_pem.extend_from_slice(key_pem.as_slice());
+            let identity = ReqwestIdentity::from_pem(cert_pem.as_slice()).map_err(|error| {
+                config_error(format!(
+                    "failed to parse client identity using {} and {}: {error}",
+                    cert_location.display(),
+                    key_location.display()
+                ))
+            })?;
+            builder = builder.identity(identity);
+        }
+        (Some(_), None) | (None, Some(_)) => {
+            return Err(config_error(
+                "client_certificate and client_private_key must both be provided for mTLS",
+            ));
+        }
+        (None, None) => {}
+    }
+
+    builder
+        .build()
+        .map_err(|error| Box::new(error) as Box<dyn Error>)
+}
+
+fn resolve_otlp_timeout(signal_var: &str) -> Duration {
+    if let Some(timeout) = read_timeout_env(signal_var) {
+        return timeout;
+    }
+    if let Some(timeout) = read_timeout_env(OTEL_EXPORTER_OTLP_TIMEOUT) {
+        return timeout;
+    }
+    OTEL_EXPORTER_OTLP_TIMEOUT_DEFAULT
+}
+
+fn read_timeout_env(var: &str) -> Option<Duration> {
+    let value = env::var(var).ok()?;
+    let parsed = value.parse::<i64>().ok()?;
+    if parsed < 0 {
+        return None;
+    }
+    Some(Duration::from_millis(parsed as u64))
+}
+
+fn read_bytes(base: &Path, provided: &PathBuf) -> Result<(Vec<u8>, PathBuf), Box<dyn Error>> {
+    let resolved = resolve_config_path(base, provided);
+    match fs::read(&resolved) {
+        Ok(bytes) => Ok((bytes, resolved)),
+        Err(error) => Err(Box::new(io::Error::new(
+            error.kind(),
+            format!("failed to read {}: {error}", resolved.display()),
+        ))),
+    }
+}
+
+fn resolve_config_path(base: &Path, provided: &PathBuf) -> PathBuf {
+    if provided.is_absolute() {
+        provided.clone()
+    } else {
+        base.join(provided)
+    }
+}
+
+fn config_error(message: impl Into<String>) -> Box<dyn Error> {
+    Box::new(io::Error::new(ErrorKind::InvalidData, message.into()))
+}

docs/config.md

@@ -651,6 +651,23 @@ Set `otel.exporter` to control where events go:
   }}
   ```
 
+Both OTLP exporters accept an optional `tls` block so you can trust a custom CA
+or enable mutual TLS. Relative paths are resolved against `~/.codex/`:
+
+```toml
+[otel]
+exporter = { otlp-http = {
+  endpoint = "https://otel.example.com/v1/logs",
+  protocol = "binary",
+  headers = { "x-otlp-api-key" = "${OTLP_TOKEN}" },
+  tls = {
+    ca-certificate = "certs/otel-ca.pem",
+    client-certificate = "/etc/codex/certs/client.pem",
+    client-private-key = "/etc/codex/certs/client-key.pem",
+  }
+}}
+```
+
 If the exporter is `none` nothing is written anywhere; otherwise you must run or point to your
 own collector. All exporters run on a background batch worker that is flushed on
 shutdown.

docs/example-config.md

@@ -369,4 +369,17 @@ exporter = "none"
 #   endpoint = "https://otel.example.com:4317",
 #   headers = { "x-otlp-meta" = "abc123" }
 # }}
+
+# Example OTLP exporter with mutual TLS
+# [otel]
+# exporter = { otlp-http = {
+#   endpoint = "https://otel.example.com/v1/logs",
+#   protocol = "binary",
+#   headers = { "x-otlp-api-key" = "${OTLP_TOKEN}" },
+#   tls = {
+#     ca-certificate = "certs/otel-ca.pem",
+#     client-certificate = "/etc/codex/certs/client.pem",
+#     client-private-key = "/etc/codex/certs/client-key.pem",
+#   }
+# }}
 ```