[codex] add otel tracing (#7844)

This commit is contained in:
Anton Panasenko authored and GitHub committed 2025-12-12 17:07:17 -08:00
1 parent 596fcd040f
commit ad7b9d63c3
39 files changed
+958 -315

No files matched your search

+1
View File
@@ -10,6 +10,7 @@ pub struct OtelSettings {
pub service_version: String,
pub codex_home: PathBuf,
pub exporter: OtelExporter,
pub trace_exporter: OtelExporter,
}
#[derive(Clone, Debug)]
+1 -23
View File
@@ -1,26 +1,4 @@
pub mod config;
pub mod otel_event_manager;
#[cfg(feature = "otel")]
pub mod otel_manager;
pub mod otel_provider;
#[cfg(not(feature = "otel"))]
mod imp {
use reqwest::header::HeaderMap;
use tracing::Span;
pub struct OtelProvider;
impl OtelProvider {
pub fn from(_settings: &crate::config::OtelSettings) -> Option<Self> {
None
}
pub fn headers(_span: &Span) -> HeaderMap {
HeaderMap::new()
}
}
}
#[cfg(not(feature = "otel"))]
pub use imp::OtelProvider;
@@ -1,5 +1,7 @@
use crate::otel_provider::traceparent_context_from_env;
use chrono::SecondsFormat;
use chrono::Utc;
use codex_api::ResponseEvent;
use codex_app_server_protocol::AuthMode;
use codex_protocol::ConversationId;
use codex_protocol::config_types::ReasoningSummary;
@@ -8,6 +10,7 @@ use codex_protocol::openai_models::ReasoningEffort;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::ReviewDecision;
use codex_protocol::protocol::SandboxPolicy;
use codex_protocol::protocol::SessionSource;
use codex_protocol::user_input::UserInput;
use eventsource_stream::Event as StreamEvent;
use eventsource_stream::EventStreamError as StreamError;
@@ -16,10 +19,14 @@ use reqwest::Response;
use serde::Serialize;
use std::borrow::Cow;
use std::fmt::Display;
use std::future::Future;
use std::time::Duration;
use std::time::Instant;
use strum_macros::Display;
use tokio::time::error::Elapsed;
use tracing::Span;
use tracing::info_span;
use tracing_opentelemetry::OpenTelemetrySpanExt;
#[derive(Debug, Clone, Serialize, Display)]
#[serde(rename_all = "snake_case")]
@@ -42,11 +49,12 @@ pub struct OtelEventMetadata {
}
#[derive(Debug, Clone)]
pub struct OtelEventManager {
pub struct OtelManager {
metadata: OtelEventMetadata,
session_span: Span,
}
impl OtelEventManager {
impl OtelManager {
#[allow(clippy::too_many_arguments)]
pub fn new(
conversation_id: ConversationId,
@@ -57,7 +65,14 @@ impl OtelEventManager {
auth_mode: Option<AuthMode>,
log_user_prompts: bool,
terminal_type: String,
) -> OtelEventManager {
session_source: SessionSource,
) -> OtelManager {
let session_span = info_span!("new_session", conversation_id = %conversation_id, session_source = %session_source);
if let Some(context) = traceparent_context_from_env() {
session_span.set_parent(context);
}
Self {
metadata: OtelEventMetadata {
conversation_id,
@@ -70,6 +85,7 @@ impl OtelEventManager {
app_version: env!("CARGO_PKG_VERSION"),
terminal_type,
},
session_span,
}
}
@@ -80,6 +96,30 @@ impl OtelEventManager {
manager
}
pub fn current_span(&self) -> &Span {
&self.session_span
}
pub fn record_responses(&self, handle_responses_span: &Span, event: &ResponseEvent) {
handle_responses_span.record("otel.name", OtelManager::responses_type(event));
match event {
ResponseEvent::OutputItemDone(item) => {
handle_responses_span.record("from", "output_item_done");
if let ResponseItem::FunctionCall { name, .. } = &item {
handle_responses_span.record("tool_name", name.as_str());
}
}
ResponseEvent::OutputItemAdded(item) => {
handle_responses_span.record("from", "output_item_added");
if let ResponseItem::FunctionCall { name, .. } = &item {
handle_responses_span.record("tool_name", name.as_str());
}
}
_ => {}
}
}
#[allow(clippy::too_many_arguments)]
pub fn conversation_starts(
&self,
@@ -394,27 +434,13 @@ impl OtelEventManager {
Err(error) => (Cow::Owned(error.to_string()), false),
};
let success_str = if success { "true" } else { "false" };
tracing::event!(
tracing::Level::INFO,
event.name = "codex.tool_result",
event.timestamp = %timestamp(),
conversation.id = %self.metadata.conversation_id,
app.version = %self.metadata.app_version,
auth_mode = self.metadata.auth_mode,
user.account_id= self.metadata.account_id,
user.email = self.metadata.account_email,
terminal.type = %self.metadata.terminal_type,
model = %self.metadata.model,
slug = %self.metadata.slug,
tool_name = %tool_name,
call_id = %call_id,
arguments = %arguments,
duration_ms = %duration.as_millis(),
success = %success_str,
// `output` is truncated by the tool layer before reaching telemetry.
output = %output,
self.tool_result(
tool_name,
call_id,
arguments,
duration,
success,
output.as_ref(),
);
result
@@ -471,6 +497,38 @@ impl OtelEventManager {
output = %output,
);
}
fn responses_type(event: &ResponseEvent) -> String {
match event {
ResponseEvent::Created => "created".into(),
ResponseEvent::OutputItemDone(item) => OtelManager::responses_item_type(item),
ResponseEvent::OutputItemAdded(item) => OtelManager::responses_item_type(item),
ResponseEvent::Completed { .. } => "completed".into(),
ResponseEvent::OutputTextDelta(_) => "text_delta".into(),
ResponseEvent::ReasoningSummaryDelta { .. } => "reasoning_summary_delta".into(),
ResponseEvent::ReasoningContentDelta { .. } => "reasoning_content_delta".into(),
ResponseEvent::ReasoningSummaryPartAdded { .. } => {
"reasoning_summary_part_added".into()
}
ResponseEvent::RateLimits(_) => "rate_limits".into(),
}
}
fn responses_item_type(item: &ResponseItem) -> String {
match item {
ResponseItem::Message { role, .. } => format!("message_from_{role}"),
ResponseItem::Reasoning { .. } => "reasoning".into(),
ResponseItem::LocalShellCall { .. } => "local_shell_call".into(),
ResponseItem::FunctionCall { .. } => "function_call".into(),
ResponseItem::FunctionCallOutput { .. } => "function_call_output".into(),
ResponseItem::CustomToolCall { .. } => "custom_tool_call".into(),
ResponseItem::CustomToolCallOutput { .. } => "custom_tool_call_output".into(),
ResponseItem::WebSearchCall { .. } => "web_search_call".into(),
ResponseItem::GhostSnapshot { .. } => "ghost_snapshot".into(),
ResponseItem::Compaction { .. } => "compaction".into(),
ResponseItem::Other => "other".into(),
}
}
}
fn timestamp() -> String {
+340 -83
View File
@@ -4,142 +4,366 @@ use crate::config::OtelSettings;
use crate::config::OtelTlsConfig;
use codex_utils_absolute_path::AbsolutePathBuf;
use http::Uri;
use opentelemetry::Context;
use opentelemetry::KeyValue;
use opentelemetry::context::ContextGuard;
use opentelemetry::global;
use opentelemetry::propagation::TextMapPropagator;
use opentelemetry::trace::TraceContextExt;
use opentelemetry::trace::TracerProvider as _;
use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge;
use opentelemetry_otlp::LogExporter;
use opentelemetry_otlp::OTEL_EXPORTER_OTLP_LOGS_TIMEOUT;
use opentelemetry_otlp::OTEL_EXPORTER_OTLP_TIMEOUT;
use opentelemetry_otlp::OTEL_EXPORTER_OTLP_TIMEOUT_DEFAULT;
use opentelemetry_otlp::OTEL_EXPORTER_OTLP_TRACES_TIMEOUT;
use opentelemetry_otlp::Protocol;
use opentelemetry_otlp::SpanExporter;
use opentelemetry_otlp::WithExportConfig;
use opentelemetry_otlp::WithHttpConfig;
use opentelemetry_otlp::WithTonicConfig;
use opentelemetry_sdk::Resource;
use opentelemetry_sdk::logs::SdkLoggerProvider;
use opentelemetry_sdk::propagation::TraceContextPropagator;
use opentelemetry_sdk::trace::BatchSpanProcessor;
use opentelemetry_sdk::trace::SdkTracerProvider;
use opentelemetry_sdk::trace::Tracer;
use opentelemetry_semantic_conventions as semconv;
use reqwest::Certificate as ReqwestCertificate;
use reqwest::Identity as ReqwestIdentity;
use reqwest::header::HeaderMap;
use reqwest::header::HeaderName;
use reqwest::header::HeaderValue;
use std::cell::RefCell;
use std::collections::HashMap;
use std::env;
use std::error::Error;
use std::fs;
use std::io::ErrorKind;
use std::io::{self};
use std::path::PathBuf;
use std::sync::OnceLock;
use std::time::Duration;
use tonic::metadata::MetadataMap;
use tonic::transport::Certificate as TonicCertificate;
use tonic::transport::ClientTlsConfig;
use tonic::transport::Identity as TonicIdentity;
use tracing::debug;
use tracing::level_filters::LevelFilter;
use tracing::warn;
use tracing_subscriber::Layer;
use tracing_subscriber::registry::LookupSpan;
const ENV_ATTRIBUTE: &str = "env";
const TRACEPARENT_ENV_VAR: &str = "TRACEPARENT";
const TRACESTATE_ENV_VAR: &str = "TRACESTATE";
static TRACEPARENT_CONTEXT: OnceLock<Option<Context>> = OnceLock::new();
thread_local! {
static TRACEPARENT_GUARD: RefCell<Option<ContextGuard>> = const { RefCell::new(None) };
}
pub struct OtelProvider {
pub logger: SdkLoggerProvider,
pub logger: Option<SdkLoggerProvider>,
pub tracer_provider: Option<SdkTracerProvider>,
pub tracer: Option<Tracer>,
}
impl OtelProvider {
pub fn shutdown(&self) {
let _ = self.logger.shutdown();
if let Some(logger) = &self.logger {
let _ = logger.shutdown();
}
if let Some(tracer_provider) = &self.tracer_provider {
let _ = tracer_provider.shutdown();
}
}
pub fn from(settings: &OtelSettings) -> Result<Option<Self>, Box<dyn Error>> {
let resource = Resource::builder()
.with_service_name(settings.service_name.clone())
.with_attributes(vec![
KeyValue::new(
semconv::attribute::SERVICE_VERSION,
settings.service_version.clone(),
),
KeyValue::new(ENV_ATTRIBUTE, settings.environment.clone()),
])
.build();
let log_enabled = !matches!(settings.exporter, OtelExporter::None);
let trace_enabled = !matches!(settings.trace_exporter, OtelExporter::None);
let mut builder = SdkLoggerProvider::builder().with_resource(resource);
if !log_enabled && !trace_enabled {
debug!("No exporter enabled in OTLP settings.");
return Ok(None);
}
match &settings.exporter {
OtelExporter::None => {
debug!("No exporter enabled in OTLP settings.");
return Ok(None);
}
OtelExporter::OtlpGrpc {
endpoint,
headers,
tls,
} => {
debug!("Using OTLP Grpc exporter: {endpoint}");
let resource = make_resource(settings);
let logger = log_enabled
.then(|| build_logger(&resource, &settings.exporter))
.transpose()?;
let mut header_map = HeaderMap::new();
for (key, value) in headers {
if let Ok(name) = HeaderName::from_bytes(key.as_bytes())
&& let Ok(val) = HeaderValue::from_str(value)
{
header_map.insert(name, val);
}
}
let tracer_provider = trace_enabled
.then(|| build_tracer_provider(&resource, &settings.trace_exporter))
.transpose()?;
let base_tls_config = ClientTlsConfig::new()
.with_enabled_roots()
.assume_http2(true);
let tracer = tracer_provider
.as_ref()
.map(|provider| provider.tracer(settings.service_name.clone()));
let tls_config = match tls.as_ref() {
Some(tls) => build_grpc_tls_config(endpoint, base_tls_config, tls)?,
None => base_tls_config,
};
let exporter = LogExporter::builder()
.with_tonic()
.with_endpoint(endpoint)
.with_metadata(MetadataMap::from_headers(header_map))
.with_tls_config(tls_config)
.build()?;
builder = builder.with_batch_exporter(exporter);
}
OtelExporter::OtlpHttp {
endpoint,
headers,
protocol,
tls,
} => {
debug!("Using OTLP Http exporter: {endpoint}");
let protocol = match protocol {
OtelHttpProtocol::Binary => Protocol::HttpBinary,
OtelHttpProtocol::Json => Protocol::HttpJson,
};
let mut exporter_builder = LogExporter::builder()
.with_http()
.with_endpoint(endpoint)
.with_protocol(protocol)
.with_headers(headers.clone());
if let Some(tls) = tls.as_ref() {
let client = build_http_client(tls)?;
exporter_builder = exporter_builder.with_http_client(client);
}
let exporter = exporter_builder.build()?;
builder = builder.with_batch_exporter(exporter);
}
if let Some(provider) = tracer_provider.clone() {
let _ = global::set_tracer_provider(provider);
global::set_text_map_propagator(TraceContextPropagator::new());
}
if tracer.is_some() {
attach_traceparent_context();
}
Ok(Some(Self {
logger: builder.build(),
logger,
tracer_provider,
tracer,
}))
}
pub fn logger_layer<S>(&self) -> Option<impl Layer<S> + Send + Sync>
where
S: tracing::Subscriber + for<'span> LookupSpan<'span> + Send + Sync,
{
self.logger.as_ref().map(|logger| {
OpenTelemetryTracingBridge::new(logger).with_filter(
tracing_subscriber::filter::filter_fn(OtelProvider::codex_export_filter),
)
})
}
pub fn tracing_layer<S>(&self) -> Option<impl Layer<S> + Send + Sync>
where
S: tracing::Subscriber + for<'span> LookupSpan<'span> + Send + Sync,
{
self.tracer.as_ref().map(|tracer| {
tracing_opentelemetry::layer()
.with_tracer(tracer.clone())
.with_filter(LevelFilter::INFO)
})
}
pub fn codex_export_filter(meta: &tracing::Metadata<'_>) -> bool {
meta.target().starts_with("codex_otel")
}
}
impl Drop for OtelProvider {
fn drop(&mut self) {
let _ = self.logger.shutdown();
if let Some(logger) = &self.logger {
let _ = logger.shutdown();
}
if let Some(tracer_provider) = &self.tracer_provider {
let _ = tracer_provider.shutdown();
}
}
}
pub(crate) fn traceparent_context_from_env() -> Option<Context> {
TRACEPARENT_CONTEXT
.get_or_init(load_traceparent_context)
.clone()
}
fn attach_traceparent_context() {
TRACEPARENT_GUARD.with(|guard| {
let mut guard = guard.borrow_mut();
if guard.is_some() {
return;
}
if let Some(context) = traceparent_context_from_env() {
*guard = Some(context.attach());
}
});
}
fn load_traceparent_context() -> Option<Context> {
let traceparent = env::var(TRACEPARENT_ENV_VAR).ok()?;
let tracestate = env::var(TRACESTATE_ENV_VAR).ok();
match extract_traceparent_context(traceparent, tracestate) {
Some(context) => {
debug!("TRACEPARENT detected; continuing trace from parent context");
Some(context)
}
None => {
warn!("TRACEPARENT is set but invalid; ignoring trace context");
None
}
}
}
fn extract_traceparent_context(traceparent: String, tracestate: Option<String>) -> Option<Context> {
let mut headers = HashMap::new();
headers.insert("traceparent".to_string(), traceparent);
if let Some(tracestate) = tracestate {
headers.insert("tracestate".to_string(), tracestate);
}
let context = TraceContextPropagator::new().extract(&headers);
let span = context.span();
let span_context = span.span_context();
if !span_context.is_valid() {
return None;
}
Some(context)
}
fn make_resource(settings: &OtelSettings) -> Resource {
Resource::builder()
.with_service_name(settings.service_name.clone())
.with_attributes(vec![
KeyValue::new(
semconv::attribute::SERVICE_VERSION,
settings.service_version.clone(),
),
KeyValue::new(ENV_ATTRIBUTE, settings.environment.clone()),
])
.build()
}
fn build_logger(
resource: &Resource,
exporter: &OtelExporter,
) -> Result<SdkLoggerProvider, Box<dyn Error>> {
let mut builder = SdkLoggerProvider::builder().with_resource(resource.clone());
match exporter {
OtelExporter::None => return Ok(builder.build()),
OtelExporter::OtlpGrpc {
endpoint,
headers,
tls,
} => {
debug!("Using OTLP Grpc exporter: {endpoint}");
let header_map = build_header_map(headers);
let base_tls_config = ClientTlsConfig::new()
.with_enabled_roots()
.assume_http2(true);
let tls_config = match tls.as_ref() {
Some(tls) => build_grpc_tls_config(endpoint, base_tls_config, tls)?,
None => base_tls_config,
};
let exporter = LogExporter::builder()
.with_tonic()
.with_endpoint(endpoint)
.with_metadata(MetadataMap::from_headers(header_map))
.with_tls_config(tls_config)
.build()?;
builder = builder.with_batch_exporter(exporter);
}
OtelExporter::OtlpHttp {
endpoint,
headers,
protocol,
tls,
} => {
debug!("Using OTLP Http exporter: {endpoint}");
let protocol = match protocol {
OtelHttpProtocol::Binary => Protocol::HttpBinary,
OtelHttpProtocol::Json => Protocol::HttpJson,
};
let mut exporter_builder = LogExporter::builder()
.with_http()
.with_endpoint(endpoint)
.with_protocol(protocol)
.with_headers(headers.clone());
if let Some(tls) = tls.as_ref() {
let client = build_http_client(tls, OTEL_EXPORTER_OTLP_LOGS_TIMEOUT)?;
exporter_builder = exporter_builder.with_http_client(client);
}
let exporter = exporter_builder.build()?;
builder = builder.with_batch_exporter(exporter);
}
}
Ok(builder.build())
}
fn build_tracer_provider(
resource: &Resource,
exporter: &OtelExporter,
) -> Result<SdkTracerProvider, Box<dyn Error>> {
let span_exporter = match exporter {
OtelExporter::None => return Ok(SdkTracerProvider::builder().build()),
OtelExporter::OtlpGrpc {
endpoint,
headers,
tls,
} => {
debug!("Using OTLP Grpc exporter for traces: {endpoint}");
let header_map = build_header_map(headers);
let base_tls_config = ClientTlsConfig::new()
.with_enabled_roots()
.assume_http2(true);
let tls_config = match tls.as_ref() {
Some(tls) => build_grpc_tls_config(endpoint, base_tls_config, tls)?,
None => base_tls_config,
};
SpanExporter::builder()
.with_tonic()
.with_endpoint(endpoint)
.with_metadata(MetadataMap::from_headers(header_map))
.with_tls_config(tls_config)
.build()?
}
OtelExporter::OtlpHttp {
endpoint,
headers,
protocol,
tls,
} => {
debug!("Using OTLP Http exporter for traces: {endpoint}");
let protocol = match protocol {
OtelHttpProtocol::Binary => Protocol::HttpBinary,
OtelHttpProtocol::Json => Protocol::HttpJson,
};
let mut exporter_builder = SpanExporter::builder()
.with_http()
.with_endpoint(endpoint)
.with_protocol(protocol)
.with_headers(headers.clone());
if let Some(tls) = tls.as_ref() {
let client = build_http_client(tls, OTEL_EXPORTER_OTLP_TRACES_TIMEOUT)?;
exporter_builder = exporter_builder.with_http_client(client);
}
exporter_builder.build()?
}
};
let processor = BatchSpanProcessor::builder(span_exporter).build();
Ok(SdkTracerProvider::builder()
.with_resource(resource.clone())
.with_span_processor(processor)
.build())
}
fn build_header_map(headers: &HashMap<String, String>) -> HeaderMap {
let mut header_map = HeaderMap::new();
for (key, value) in headers {
if let Ok(name) = HeaderName::from_bytes(key.as_bytes())
&& let Ok(val) = HeaderValue::from_str(value)
{
header_map.insert(name, val);
}
}
header_map
}
fn build_grpc_tls_config(
endpoint: &str,
tls_config: ClientTlsConfig,
@@ -182,17 +406,21 @@ fn build_grpc_tls_config(
/// `opentelemetry_sdk` `BatchLogProcessor` spawns a dedicated OS thread that uses
/// `futures_executor::block_on()` rather than tokio. When the async reqwest client's
/// timeout calls `tokio::time::sleep()`, it panics with "no reactor running".
fn build_http_client(tls: &OtelTlsConfig) -> Result<reqwest::blocking::Client, Box<dyn Error>> {
fn build_http_client(
tls: &OtelTlsConfig,
timeout_var: &str,
) -> Result<reqwest::blocking::Client, Box<dyn Error>> {
// Wrap in block_in_place because reqwest::blocking::Client creates its own
// internal tokio runtime, which would panic if built directly from an async context.
tokio::task::block_in_place(|| build_http_client_inner(tls))
tokio::task::block_in_place(|| build_http_client_inner(tls, timeout_var))
}
fn build_http_client_inner(
tls: &OtelTlsConfig,
timeout_var: &str,
) -> Result<reqwest::blocking::Client, Box<dyn Error>> {
let mut builder = reqwest::blocking::Client::builder()
.timeout(resolve_otlp_timeout(OTEL_EXPORTER_OTLP_LOGS_TIMEOUT));
let mut builder =
reqwest::blocking::Client::builder().timeout(resolve_otlp_timeout(timeout_var));
if let Some(path) = tls.ca_certificate.as_ref() {
let (pem, location) = read_bytes(path)?;
@@ -267,3 +495,32 @@ fn read_bytes(path: &AbsolutePathBuf) -> Result<(Vec<u8>, PathBuf), Box<dyn Erro
fn config_error(message: impl Into<String>) -> Box<dyn Error> {
Box::new(io::Error::new(ErrorKind::InvalidData, message.into()))
}
#[cfg(test)]
mod tests {
use super::*;
use opentelemetry::trace::SpanId;
use opentelemetry::trace::TraceContextExt;
use opentelemetry::trace::TraceId;
#[test]
fn parses_valid_traceparent() {
let trace_id = "00000000000000000000000000000001";
let span_id = "0000000000000002";
let context = extract_traceparent_context(format!("00-{trace_id}-{span_id}-01"), None)
.expect("trace context");
let span = context.span();
let span_context = span.span_context();
assert_eq!(
span_context.trace_id(),
TraceId::from_hex(trace_id).unwrap()
);
assert_eq!(span_context.span_id(), SpanId::from_hex(span_id).unwrap());
assert!(span_context.is_remote());
}
#[test]
fn invalid_traceparent_returns_none() {
assert!(extract_traceparent_context("not-a-traceparent".to_string(), None).is_none());
}
}