feat: show runtime metrics in console (#10278)

Summary of changes:

- Adds a new feature flag: runtime_metrics
  - Declared in core/src/features.rs
  - Added to core/config.schema.json
  - Wired into OTEL init in core/src/otel_init.rs

- Enables on-demand runtime metric snapshots in OTEL
  - Adds runtime_metrics: bool to otel/src/config.rs
  - Enables experimental custom reader features in otel/Cargo.toml
  - Adds snapshot/reset/summary APIs in:
    - otel/src/lib.rs
    - otel/src/metrics/client.rs
    - otel/src/metrics/config.rs
    - otel/src/metrics/error.rs

- Defines metric names and a runtime summary builder
  - New files:
    - otel/src/metrics/names.rs
    - otel/src/metrics/runtime_metrics.rs
  - Summarizes totals for:
    - Tool calls
    - API requests
    - SSE/streaming events

- Instruments metrics collection in OTEL manager
  - otel/src/traces/otel_manager.rs now records:
    - API call counts + durations
    - SSE event counts + durations (success/failure)
    - Tool call metrics now use shared constants

- Surfaces runtime metrics in the TUI
  - Resets runtime metrics at turn start in tui/src/chatwidget.rs
- Displays metrics in the final separator line in
tui/src/history_cell.rs

- Adds tests
  - New OTEL tests:
    - otel/tests/suite/snapshot.rs
    - otel/tests/suite/runtime_summary.rs
  - New TUI test:
- final_message_separator_includes_runtime_metrics in
tui/src/history_cell.rs

Scope:
- 19 files changed
- ~652 insertions, 38 deletions


<img width="922" height="169" alt="Screenshot 2026-01-30 at 4 11 34 PM"
src="https://github.com/user-attachments/assets/1efd754d-a16d-4564-83a5-f4442fd2f998"
/>

codex-rs/otel/src/config.rs

@@ -37,6 +37,7 @@ pub struct OtelSettings {
     pub exporter: OtelExporter,
     pub trace_exporter: OtelExporter,
     pub metrics_exporter: OtelExporter,
+    pub runtime_metrics: bool,
 }
 
 #[derive(Clone, Debug)]

codex-rs/otel/src/lib.rs

@@ -14,9 +14,14 @@ use crate::metrics::validation::validate_tag_key;
 use crate::metrics::validation::validate_tag_value;
 use crate::otel_provider::OtelProvider;
 use codex_protocol::ThreadId;
+use opentelemetry_sdk::metrics::data::ResourceMetrics;
 use serde::Serialize;
 use std::time::Duration;
 use strum_macros::Display;
+use tracing::debug;
+
+pub use crate::metrics::runtime_metrics::RuntimeMetricTotals;
+pub use crate::metrics::runtime_metrics::RuntimeMetricsSummary;
 
 #[derive(Debug, Clone, Serialize, Display)]
 #[serde(rename_all = "snake_case")]
@@ -137,6 +142,39 @@ impl OtelManager {
         metrics.shutdown()
     }
 
+    pub fn snapshot_metrics(&self) -> MetricsResult<ResourceMetrics> {
+        let Some(metrics) = &self.metrics else {
+            return Err(MetricsError::ExporterDisabled);
+        };
+        metrics.snapshot()
+    }
+
+    /// Collect and discard a runtime metrics snapshot to reset delta accumulators.
+    pub fn reset_runtime_metrics(&self) {
+        if self.metrics.is_none() {
+            return;
+        }
+        if let Err(err) = self.snapshot_metrics() {
+            debug!("runtime metrics reset skipped: {err}");
+        }
+    }
+
+    /// Collect a runtime metrics summary if debug snapshots are available.
+    pub fn runtime_metrics_summary(&self) -> Option<RuntimeMetricsSummary> {
+        let snapshot = match self.snapshot_metrics() {
+            Ok(snapshot) => snapshot,
+            Err(_) => {
+                return None;
+            }
+        };
+        let summary = RuntimeMetricsSummary::from_snapshot(&snapshot);
+        if summary.is_empty() {
+            None
+        } else {
+            Some(summary)
+        }
+    }
+
     fn tags_with_metadata<'a>(
         &'a self,
         tags: &'a [(&'a str, &'a str)],

codex-rs/otel/src/metrics/client.rs

@@ -22,13 +22,20 @@ use opentelemetry_otlp::WithTonicConfig;
 use opentelemetry_otlp::tonic_types::metadata::MetadataMap;
 use opentelemetry_otlp::tonic_types::transport::ClientTlsConfig;
 use opentelemetry_sdk::Resource;
+use opentelemetry_sdk::metrics::InstrumentKind;
+use opentelemetry_sdk::metrics::ManualReader;
 use opentelemetry_sdk::metrics::PeriodicReader;
+use opentelemetry_sdk::metrics::Pipeline;
 use opentelemetry_sdk::metrics::SdkMeterProvider;
 use opentelemetry_sdk::metrics::Temporality;
+use opentelemetry_sdk::metrics::data::ResourceMetrics;
+use opentelemetry_sdk::metrics::reader::MetricReader;
 use opentelemetry_semantic_conventions as semconv;
 use std::collections::BTreeMap;
 use std::collections::HashMap;
+use std::sync::Arc;
 use std::sync::Mutex;
+use std::sync::Weak;
 use std::time::Duration;
 use tracing::debug;
 
@@ -37,6 +44,39 @@ const METER_NAME: &str = "codex";
 const DURATION_UNIT: &str = "ms";
 const DURATION_DESCRIPTION: &str = "Duration in milliseconds.";
 
+#[derive(Clone, Debug)]
+struct SharedManualReader {
+    inner: Arc<ManualReader>,
+}
+
+impl SharedManualReader {
+    fn new(inner: Arc<ManualReader>) -> Self {
+        Self { inner }
+    }
+}
+
+impl MetricReader for SharedManualReader {
+    fn register_pipeline(&self, pipeline: Weak<Pipeline>) {
+        self.inner.register_pipeline(pipeline);
+    }
+
+    fn collect(&self, rm: &mut ResourceMetrics) -> opentelemetry_sdk::error::OTelSdkResult {
+        self.inner.collect(rm)
+    }
+
+    fn force_flush(&self) -> opentelemetry_sdk::error::OTelSdkResult {
+        self.inner.force_flush()
+    }
+
+    fn shutdown_with_timeout(&self, timeout: Duration) -> opentelemetry_sdk::error::OTelSdkResult {
+        self.inner.shutdown_with_timeout(timeout)
+    }
+
+    fn temporality(&self, kind: InstrumentKind) -> Temporality {
+        self.inner.temporality(kind)
+    }
+}
+
 #[derive(Debug)]
 struct MetricsClientInner {
     meter_provider: SdkMeterProvider,
@@ -44,6 +84,7 @@ struct MetricsClientInner {
     counters: Mutex<HashMap<String, Counter<u64>>>,
     histograms: Mutex<HashMap<String, Histogram<f64>>>,
     duration_histograms: Mutex<HashMap<String, Histogram<f64>>>,
+    runtime_reader: Option<Arc<ManualReader>>,
     default_tags: BTreeMap<String, String>,
 }
 
@@ -144,26 +185,41 @@ pub struct MetricsClient(std::sync::Arc<MetricsClientInner>);
 impl MetricsClient {
     /// Build a metrics client from configuration and validate defaults.
     pub fn new(config: MetricsConfig) -> Result<Self> {
-        validate_tags(&config.default_tags)?;
+        let MetricsConfig {
+            environment,
+            service_name,
+            service_version,
+            exporter,
+            export_interval,
+            runtime_reader,
+            default_tags,
+        } = config;
+
+        validate_tags(&default_tags)?;
 
         let resource = Resource::builder()
-            .with_service_name(config.service_name.clone())
+            .with_service_name(service_name)
             .with_attributes(vec![
-                KeyValue::new(
-                    semconv::attribute::SERVICE_VERSION,
-                    config.service_version.clone(),
-                ),
-                KeyValue::new(ENV_ATTRIBUTE, config.environment.clone()),
+                KeyValue::new(semconv::attribute::SERVICE_VERSION, service_version),
+                KeyValue::new(ENV_ATTRIBUTE, environment),
             ])
             .build();
 
-        let (meter_provider, meter) = match config.exporter {
+        let runtime_reader = runtime_reader.then(|| {
+            Arc::new(
+                ManualReader::builder()
+                    .with_temporality(Temporality::Delta)
+                    .build(),
+            )
+        });
+
+        let (meter_provider, meter) = match exporter {
             MetricsExporter::InMemory(exporter) => {
-                build_provider(resource, exporter, config.export_interval)
+                build_provider(resource, exporter, export_interval, runtime_reader.clone())
             }
             MetricsExporter::Otlp(exporter) => {
                 let exporter = build_otlp_metric_exporter(exporter, Temporality::Delta)?;
-                build_provider(resource, exporter, config.export_interval)
+                build_provider(resource, exporter, export_interval, runtime_reader.clone())
             }
         };
 
@@ -173,7 +229,8 @@ impl MetricsClient {
             counters: Mutex::new(HashMap::new()),
             histograms: Mutex::new(HashMap::new()),
             duration_histograms: Mutex::new(HashMap::new()),
-            default_tags: config.default_tags,
+            runtime_reader,
+            default_tags,
         })))
     }
 
@@ -209,6 +266,18 @@ impl MetricsClient {
         Ok(Timer::new(name, tags, self))
     }
 
+    /// Collect a runtime metrics snapshot without shutting down the provider.
+    pub fn snapshot(&self) -> Result<ResourceMetrics> {
+        let Some(reader) = &self.0.runtime_reader else {
+            return Err(MetricsError::RuntimeSnapshotUnavailable);
+        };
+        let mut snapshot = ResourceMetrics::default();
+        reader
+            .collect(&mut snapshot)
+            .map_err(|source| MetricsError::RuntimeSnapshotCollect { source })?;
+        Ok(snapshot)
+    }
+
     /// Flush metrics and stop the underlying OTEL meter provider.
     pub fn shutdown(&self) -> Result<()> {
         self.0.shutdown()
@@ -219,6 +288,7 @@ fn build_provider<E>(
     resource: Resource,
     exporter: E,
     interval: Option<Duration>,
+    runtime_reader: Option<Arc<ManualReader>>,
 ) -> (SdkMeterProvider, Meter)
 where
     E: opentelemetry_sdk::metrics::exporter::PushMetricExporter + 'static,
@@ -228,10 +298,11 @@ where
         reader_builder = reader_builder.with_interval(interval);
     }
     let reader = reader_builder.build();
-    let provider = SdkMeterProvider::builder()
-        .with_resource(resource)
-        .with_reader(reader)
-        .build();
+    let mut provider_builder = SdkMeterProvider::builder().with_resource(resource);
+    if let Some(reader) = runtime_reader {
+        provider_builder = provider_builder.with_reader(SharedManualReader::new(reader));
+    }
+    let provider = provider_builder.with_reader(reader).build();
     let meter = provider.meter(METER_NAME);
     (provider, meter)
 }

codex-rs/otel/src/metrics/config.rs

@@ -19,6 +19,7 @@ pub struct MetricsConfig {
     pub(crate) service_version: String,
     pub(crate) exporter: MetricsExporter,
     pub(crate) export_interval: Option<Duration>,
+    pub(crate) runtime_reader: bool,
     pub(crate) default_tags: BTreeMap<String, String>,
 }
 
@@ -35,6 +36,7 @@ impl MetricsConfig {
             service_version: service_version.into(),
             exporter: MetricsExporter::Otlp(exporter),
             export_interval: None,
+            runtime_reader: false,
             default_tags: BTreeMap::new(),
         }
     }
@@ -52,6 +54,7 @@ impl MetricsConfig {
             service_version: service_version.into(),
             exporter: MetricsExporter::InMemory(exporter),
             export_interval: None,
+            runtime_reader: false,
             default_tags: BTreeMap::new(),
         }
     }
@@ -62,6 +65,12 @@ impl MetricsConfig {
         self
     }
 
+    /// Enable a manual reader for on-demand runtime snapshots.
+    pub fn with_runtime_reader(mut self) -> Self {
+        self.runtime_reader = true;
+        self
+    }
+
     /// Add a default tag that will be sent with every metric.
     pub fn with_tag(mut self, key: impl Into<String>, value: impl Into<String>) -> Result<Self> {
         let key = key.into();

codex-rs/otel/src/metrics/error.rs

@@ -34,4 +34,13 @@ pub enum MetricsError {
         #[source]
         source: opentelemetry_sdk::error::OTelSdkError,
     },
+
+    #[error("runtime metrics snapshot reader is not enabled")]
+    RuntimeSnapshotUnavailable,
+
+    #[error("failed to collect runtime metrics snapshot from metrics reader")]
+    RuntimeSnapshotCollect {
+        #[source]
+        source: opentelemetry_sdk::error::OTelSdkError,
+    },
 }

codex-rs/otel/src/metrics/mod.rs

@@ -1,6 +1,8 @@
 mod client;
 mod config;
 mod error;
+pub(crate) mod names;
+pub(crate) mod runtime_metrics;
 pub(crate) mod timer;
 pub(crate) mod validation;
 

codex-rs/otel/src/metrics/names.rs

@@ -0,0 +1,6 @@
+pub(crate) const TOOL_CALL_COUNT_METRIC: &str = "codex.tool.call";
+pub(crate) const TOOL_CALL_DURATION_METRIC: &str = "codex.tool.call.duration_ms";
+pub(crate) const API_CALL_COUNT_METRIC: &str = "codex.api_request";
+pub(crate) const API_CALL_DURATION_METRIC: &str = "codex.api_request.duration_ms";
+pub(crate) const SSE_EVENT_COUNT_METRIC: &str = "codex.sse_event";
+pub(crate) const SSE_EVENT_DURATION_METRIC: &str = "codex.sse_event.duration_ms";

codex-rs/otel/src/metrics/runtime_metrics.rs

@@ -0,0 +1,101 @@
+use crate::metrics::names::API_CALL_COUNT_METRIC;
+use crate::metrics::names::API_CALL_DURATION_METRIC;
+use crate::metrics::names::SSE_EVENT_COUNT_METRIC;
+use crate::metrics::names::SSE_EVENT_DURATION_METRIC;
+use crate::metrics::names::TOOL_CALL_COUNT_METRIC;
+use crate::metrics::names::TOOL_CALL_DURATION_METRIC;
+use opentelemetry_sdk::metrics::data::AggregatedMetrics;
+use opentelemetry_sdk::metrics::data::Metric;
+use opentelemetry_sdk::metrics::data::MetricData;
+use opentelemetry_sdk::metrics::data::ResourceMetrics;
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct RuntimeMetricTotals {
+    pub count: u64,
+    pub duration_ms: u64,
+}
+
+impl RuntimeMetricTotals {
+    pub fn is_empty(self) -> bool {
+        self.count == 0 && self.duration_ms == 0
+    }
+}
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct RuntimeMetricsSummary {
+    pub tool_calls: RuntimeMetricTotals,
+    pub api_calls: RuntimeMetricTotals,
+    pub streaming_events: RuntimeMetricTotals,
+}
+
+impl RuntimeMetricsSummary {
+    pub fn is_empty(self) -> bool {
+        self.tool_calls.is_empty() && self.api_calls.is_empty() && self.streaming_events.is_empty()
+    }
+
+    pub(crate) fn from_snapshot(snapshot: &ResourceMetrics) -> Self {
+        let tool_calls = RuntimeMetricTotals {
+            count: sum_counter(snapshot, TOOL_CALL_COUNT_METRIC),
+            duration_ms: sum_histogram_ms(snapshot, TOOL_CALL_DURATION_METRIC),
+        };
+        let api_calls = RuntimeMetricTotals {
+            count: sum_counter(snapshot, API_CALL_COUNT_METRIC),
+            duration_ms: sum_histogram_ms(snapshot, API_CALL_DURATION_METRIC),
+        };
+        let streaming_events = RuntimeMetricTotals {
+            count: sum_counter(snapshot, SSE_EVENT_COUNT_METRIC),
+            duration_ms: sum_histogram_ms(snapshot, SSE_EVENT_DURATION_METRIC),
+        };
+        Self {
+            tool_calls,
+            api_calls,
+            streaming_events,
+        }
+    }
+}
+
+fn sum_counter(snapshot: &ResourceMetrics, name: &str) -> u64 {
+    snapshot
+        .scope_metrics()
+        .flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics)
+        .filter(|metric| metric.name() == name)
+        .map(sum_counter_metric)
+        .sum()
+}
+
+fn sum_counter_metric(metric: &Metric) -> u64 {
+    match metric.data() {
+        AggregatedMetrics::U64(MetricData::Sum(sum)) => sum
+            .data_points()
+            .map(opentelemetry_sdk::metrics::data::SumDataPoint::value)
+            .sum(),
+        _ => 0,
+    }
+}
+
+fn sum_histogram_ms(snapshot: &ResourceMetrics, name: &str) -> u64 {
+    snapshot
+        .scope_metrics()
+        .flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics)
+        .filter(|metric| metric.name() == name)
+        .map(sum_histogram_metric_ms)
+        .sum()
+}
+
+fn sum_histogram_metric_ms(metric: &Metric) -> u64 {
+    match metric.data() {
+        AggregatedMetrics::F64(MetricData::Histogram(histogram)) => histogram
+            .data_points()
+            .map(|point| f64_to_u64(point.sum()))
+            .sum(),
+        _ => 0,
+    }
+}
+
+fn f64_to_u64(value: f64) -> u64 {
+    if !value.is_finite() || value <= 0.0 {
+        return 0;
+    }
+    let clamped = value.min(u64::MAX as f64);
+    clamped.round() as u64
+}

codex-rs/otel/src/otel_provider.rs

@@ -75,12 +75,16 @@ impl OtelProvider {
         let metrics = if matches!(metric_exporter, OtelExporter::None) {
             None
         } else {
-            Some(MetricsClient::new(MetricsConfig::otlp(
+            let mut config = MetricsConfig::otlp(
                 settings.environment.clone(),
                 settings.service_name.clone(),
                 settings.service_version.clone(),
                 metric_exporter,
-            ))?)
+            );
+            if settings.runtime_metrics {
+                config = config.with_runtime_reader();
+            }
+            Some(MetricsClient::new(config)?)
         };
 
         if let Some(metrics) = metrics.as_ref() {

codex-rs/otel/src/traces/otel_manager.rs

@@ -1,3 +1,9 @@
+use crate::metrics::names::API_CALL_COUNT_METRIC;
+use crate::metrics::names::API_CALL_DURATION_METRIC;
+use crate::metrics::names::SSE_EVENT_COUNT_METRIC;
+use crate::metrics::names::SSE_EVENT_DURATION_METRIC;
+use crate::metrics::names::TOOL_CALL_COUNT_METRIC;
+use crate::metrics::names::TOOL_CALL_DURATION_METRIC;
 use crate::otel_provider::traceparent_context_from_env;
 use chrono::SecondsFormat;
 use chrono::Utc;
@@ -29,6 +35,8 @@ pub use crate::OtelEventMetadata;
 pub use crate::OtelManager;
 pub use crate::ToolDecisionSource;
 
+const SSE_UNKNOWN_KIND: &str = "unknown";
+
 impl OtelManager {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
@@ -148,6 +156,21 @@ impl OtelManager {
         error: Option<&str>,
         duration: Duration,
     ) {
+        let success = status.is_some_and(|code| (200..=299).contains(&code)) && error.is_none();
+        let success_str = if success { "true" } else { "false" };
+        let status_str = status
+            .map(|code| code.to_string())
+            .unwrap_or_else(|| "none".to_string());
+        self.counter(
+            API_CALL_COUNT_METRIC,
+            1,
+            &[("status", status_str.as_str()), ("success", success_str)],
+        );
+        self.record_duration(
+            API_CALL_DURATION_METRIC,
+            duration,
+            &[("status", status_str.as_str()), ("success", success_str)],
+        );
         tracing::event!(
             tracing::Level::INFO,
             event.name = "codex.api_request",
@@ -215,6 +238,16 @@ impl OtelManager {
     }
 
     fn sse_event(&self, kind: &str, duration: Duration) {
+        self.counter(
+            SSE_EVENT_COUNT_METRIC,
+            1,
+            &[("kind", kind), ("success", "true")],
+        );
+        self.record_duration(
+            SSE_EVENT_DURATION_METRIC,
+            duration,
+            &[("kind", kind), ("success", "true")],
+        );
         tracing::event!(
             tracing::Level::INFO,
             event.name = "codex.sse_event",
@@ -236,6 +269,17 @@ impl OtelManager {
     where
         T: Display,
     {
+        let kind_str = kind.map_or(SSE_UNKNOWN_KIND, String::as_str);
+        self.counter(
+            SSE_EVENT_COUNT_METRIC,
+            1,
+            &[("kind", kind_str), ("success", "false")],
+        );
+        self.record_duration(
+            SSE_EVENT_DURATION_METRIC,
+            duration,
+            &[("kind", kind_str), ("success", "false")],
+        );
         match kind {
             Some(kind) => tracing::event!(
                 tracing::Level::INFO,
@@ -443,12 +487,12 @@ impl OtelManager {
     ) {
         let success_str = if success { "true" } else { "false" };
         self.counter(
-            "codex.tool.call",
+            TOOL_CALL_COUNT_METRIC,
             1,
             &[("tool", tool_name), ("success", success_str)],
         );
         self.record_duration(
-            "codex.tool.call.duration_ms",
+            TOOL_CALL_DURATION_METRIC,
             duration,
             &[("tool", tool_name), ("success", success_str)],
         );