mirror of
https://github.com/openai/codex.git
synced 2026-11-13 14:46:14 +00:00
Export turn cost as an OTEL metric (#40488)
## What changed - Emit `codex.turn.cost_microusd` as a counter with turn, conversation, interruption, speed, and reasoning-effort attributes. Convert the estimated USD string to microdollars, rounding to the nearest microdollar and skipping values that cannot be represented safely. - Start turn-cost collection when an OTLP metrics exporter is configured, even when the OTEL log exporter is disabled. ## Testing - Verify cost conversion, rounding, and metric attributes with an in-memory metrics snapshot. - Verify that a metrics-only OTLP configuration starts the turn-cost worker. GitOrigin-RevId: e60587079f097f10cc1b6d1bf889d3f7a0dc3d5f
This commit is contained in:
1 parent
fb9311db5c
commit
2497518490
7 files changed
+151
-2
No files matched your search
@@ -95,10 +95,16 @@ enum BackendAvailability {
|
|||||||
|
|
||||||
impl TurnCostWorker {
|
impl TurnCostWorker {
|
||||||
pub(crate) fn spawn(config: Arc<Config>, auth_manager: Arc<AuthManager>) -> Option<Self> {
|
pub(crate) fn spawn(config: Arc<Config>, auth_manager: Arc<AuthManager>) -> Option<Self> {
|
||||||
if !matches!(
|
let has_otel_log_exporter = matches!(
|
||||||
config.otel.exporter,
|
config.otel.exporter,
|
||||||
OtelExporterKind::OtlpHttp { .. } | OtelExporterKind::OtlpGrpc { .. }
|
OtelExporterKind::OtlpHttp { .. } | OtelExporterKind::OtlpGrpc { .. }
|
||||||
) || config.model_provider.is_amazon_bedrock()
|
);
|
||||||
|
let has_otel_metrics_exporter = matches!(
|
||||||
|
config.otel.metrics_exporter,
|
||||||
|
OtelExporterKind::OtlpHttp { .. } | OtelExporterKind::OtlpGrpc { .. }
|
||||||
|
);
|
||||||
|
if !(has_otel_log_exporter || has_otel_metrics_exporter)
|
||||||
|
|| config.model_provider.is_amazon_bedrock()
|
||||||
{
|
{
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,6 +21,39 @@ use wiremock::matchers::path;
|
|||||||
|
|
||||||
const TURN_COST_PATH: &str = "/v1/analytics/codex/turn-costs";
|
const TURN_COST_PATH: &str = "/v1/analytics/codex/turn-costs";
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn worker_starts_with_otlp_metrics_exporter_without_log_exporter() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
Mock::given(method("POST"))
|
||||||
|
.and(path(TURN_COST_PATH))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||||
|
"turns": []
|
||||||
|
})))
|
||||||
|
.expect(1)
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
let codex_home = TempDir::new().expect("temporary Codex home");
|
||||||
|
let mut config = ConfigBuilder::default()
|
||||||
|
.codex_home(codex_home.path().to_path_buf())
|
||||||
|
.build()
|
||||||
|
.await
|
||||||
|
.expect("test config");
|
||||||
|
config.chatgpt_base_url = server.uri();
|
||||||
|
config.otel.exporter = OtelExporterKind::None;
|
||||||
|
config.otel.metrics_exporter = OtelExporterKind::OtlpGrpc {
|
||||||
|
endpoint: server.uri(),
|
||||||
|
headers: HashMap::new(),
|
||||||
|
tls: None,
|
||||||
|
};
|
||||||
|
let auth_manager = AuthManager::from_auth_for_testing(CodexAuth::from_api_key("sk-test"));
|
||||||
|
|
||||||
|
let worker = TurnCostWorker::spawn(Arc::new(config), auth_manager)
|
||||||
|
.expect("OTLP metrics exporter should enable turn-cost collection");
|
||||||
|
wait_for_request_count(&server, /*expected*/ 1).await;
|
||||||
|
worker.shutdown();
|
||||||
|
server.verify().await;
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn handle_observes_only_matching_model_provider() {
|
async fn handle_observes_only_matching_model_provider() {
|
||||||
let codex_home = TempDir::new().expect("temporary Codex home");
|
let codex_home = TempDir::new().expect("temporary Codex home");
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ use crate::metrics::STARTUP_PHASE_DURATION_METRIC;
|
|||||||
use crate::metrics::SessionMetricTagValues;
|
use crate::metrics::SessionMetricTagValues;
|
||||||
use crate::metrics::TOOL_CALL_COUNT_METRIC;
|
use crate::metrics::TOOL_CALL_COUNT_METRIC;
|
||||||
use crate::metrics::TOOL_CALL_DURATION_METRIC;
|
use crate::metrics::TOOL_CALL_DURATION_METRIC;
|
||||||
|
use crate::metrics::TURN_COST_MICROUSD_METRIC;
|
||||||
use crate::metrics::TURN_TTFT_DURATION_METRIC;
|
use crate::metrics::TURN_TTFT_DURATION_METRIC;
|
||||||
use crate::metrics::WEBSOCKET_EVENT_COUNT_METRIC;
|
use crate::metrics::WEBSOCKET_EVENT_COUNT_METRIC;
|
||||||
use crate::metrics::WEBSOCKET_EVENT_DURATION_METRIC;
|
use crate::metrics::WEBSOCKET_EVENT_DURATION_METRIC;
|
||||||
@@ -281,6 +282,45 @@ impl SessionTelemetry {
|
|||||||
speed: Option<&str>,
|
speed: Option<&str>,
|
||||||
reasoning_effort: Option<&str>,
|
reasoning_effort: Option<&str>,
|
||||||
) {
|
) {
|
||||||
|
let (dollars, fractional) = estimated_usd.split_once('.').unwrap_or((estimated_usd, ""));
|
||||||
|
let fractional = fractional.as_bytes();
|
||||||
|
let fractional_precision = 6_usize;
|
||||||
|
let estimated_microusd = dollars.parse::<u64>().ok().and_then(|dollars| {
|
||||||
|
if !fractional.iter().all(u8::is_ascii_digit) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let fractional_microusd = fractional
|
||||||
|
.iter()
|
||||||
|
.take(fractional_precision)
|
||||||
|
.fold(0_u64, |value, digit| value * 10 + u64::from(digit - b'0'))
|
||||||
|
* 10_u64.pow(fractional_precision.saturating_sub(fractional.len()) as u32);
|
||||||
|
let round_up = fractional
|
||||||
|
.get(fractional_precision)
|
||||||
|
.is_some_and(|digit| *digit >= b'5');
|
||||||
|
let estimated_microusd = dollars
|
||||||
|
.checked_mul(1_000_000)?
|
||||||
|
.checked_add(fractional_microusd)?
|
||||||
|
.checked_add(u64::from(round_up))?;
|
||||||
|
i64::try_from(estimated_microusd).ok()
|
||||||
|
});
|
||||||
|
if let Some(estimated_microusd) = estimated_microusd {
|
||||||
|
let conversation_id = self.metadata.conversation_id.to_string();
|
||||||
|
let mut tags = vec![
|
||||||
|
("turn.id", turn_id),
|
||||||
|
("conversation.id", conversation_id.as_str()),
|
||||||
|
(
|
||||||
|
"turn.interrupted",
|
||||||
|
if interrupted { "true" } else { "false" },
|
||||||
|
),
|
||||||
|
];
|
||||||
|
if let Some(speed) = speed {
|
||||||
|
tags.push(("speed", speed));
|
||||||
|
}
|
||||||
|
if let Some(reasoning_effort) = reasoning_effort {
|
||||||
|
tags.push(("reasoning_effort", reasoning_effort));
|
||||||
|
}
|
||||||
|
self.counter(TURN_COST_MICROUSD_METRIC, estimated_microusd, &tags);
|
||||||
|
}
|
||||||
log_event!(
|
log_event!(
|
||||||
self,
|
self,
|
||||||
event.name = "codex.turn_cost",
|
event.name = "codex.turn_cost",
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ use crate::metrics::names::RESPONSES_API_ENGINE_SERVICE_TBT_DURATION_METRIC;
|
|||||||
use crate::metrics::names::RESPONSES_API_ENGINE_SERVICE_TTFT_DURATION_METRIC;
|
use crate::metrics::names::RESPONSES_API_ENGINE_SERVICE_TTFT_DURATION_METRIC;
|
||||||
use crate::metrics::names::TOOL_CALL_COUNT_METRIC;
|
use crate::metrics::names::TOOL_CALL_COUNT_METRIC;
|
||||||
use crate::metrics::names::TOOL_CALL_DURATION_METRIC;
|
use crate::metrics::names::TOOL_CALL_DURATION_METRIC;
|
||||||
|
use crate::metrics::names::TURN_COST_MICROUSD_METRIC;
|
||||||
use crate::metrics::names::TURN_TOKEN_USAGE_METRIC;
|
use crate::metrics::names::TURN_TOKEN_USAGE_METRIC;
|
||||||
use crate::metrics::validation::validate_tag_key;
|
use crate::metrics::validation::validate_tag_key;
|
||||||
use crate::metrics::validation::validate_tag_value;
|
use crate::metrics::validation::validate_tag_value;
|
||||||
@@ -27,6 +28,7 @@ const STATSIG_DISABLED_METRICS: &[&str] = &[
|
|||||||
RESPONSES_API_ENGINE_SERVICE_TTFT_DURATION_METRIC,
|
RESPONSES_API_ENGINE_SERVICE_TTFT_DURATION_METRIC,
|
||||||
TOOL_CALL_COUNT_METRIC,
|
TOOL_CALL_COUNT_METRIC,
|
||||||
TOOL_CALL_DURATION_METRIC,
|
TOOL_CALL_DURATION_METRIC,
|
||||||
|
TURN_COST_MICROUSD_METRIC,
|
||||||
TURN_TOKEN_USAGE_METRIC,
|
TURN_TOKEN_USAGE_METRIC,
|
||||||
];
|
];
|
||||||
|
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ pub const TURN_NETWORK_PROXY_METRIC: &str = "codex.turn.network_proxy";
|
|||||||
pub const TURN_MEMORY_METRIC: &str = "codex.turn.memory";
|
pub const TURN_MEMORY_METRIC: &str = "codex.turn.memory";
|
||||||
pub const TURN_TOOL_CALL_METRIC: &str = "codex.turn.tool.call";
|
pub const TURN_TOOL_CALL_METRIC: &str = "codex.turn.tool.call";
|
||||||
pub const TURN_TOKEN_USAGE_METRIC: &str = "codex.turn.token_usage";
|
pub const TURN_TOKEN_USAGE_METRIC: &str = "codex.turn.token_usage";
|
||||||
|
pub const TURN_COST_MICROUSD_METRIC: &str = "codex.turn.cost_microusd";
|
||||||
pub const TURN_UNIFIED_EXEC_RUNNING_PROCESSES_METRIC: &str =
|
pub const TURN_UNIFIED_EXEC_RUNNING_PROCESSES_METRIC: &str =
|
||||||
"codex.turn.unified_exec.running_processes";
|
"codex.turn.unified_exec.running_processes";
|
||||||
pub const GUARDIAN_REVIEW_COUNT_METRIC: &str = "codex.guardian.review";
|
pub const GUARDIAN_REVIEW_COUNT_METRIC: &str = "codex.guardian.review";
|
||||||
|
|||||||
@@ -604,6 +604,7 @@ mod tests {
|
|||||||
use crate::metrics::RESPONSES_API_ENGINE_SERVICE_TTFT_DURATION_METRIC;
|
use crate::metrics::RESPONSES_API_ENGINE_SERVICE_TTFT_DURATION_METRIC;
|
||||||
use crate::metrics::TOOL_CALL_COUNT_METRIC;
|
use crate::metrics::TOOL_CALL_COUNT_METRIC;
|
||||||
use crate::metrics::TOOL_CALL_DURATION_METRIC;
|
use crate::metrics::TOOL_CALL_DURATION_METRIC;
|
||||||
|
use crate::metrics::TURN_COST_MICROUSD_METRIC;
|
||||||
use crate::metrics::TURN_TOKEN_USAGE_METRIC;
|
use crate::metrics::TURN_TOKEN_USAGE_METRIC;
|
||||||
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
|
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
|
||||||
use pretty_assertions::assert_eq;
|
use pretty_assertions::assert_eq;
|
||||||
@@ -741,6 +742,7 @@ mod tests {
|
|||||||
)?;
|
)?;
|
||||||
metrics.counter(TOOL_CALL_COUNT_METRIC, /*inc*/ 1, &[])?;
|
metrics.counter(TOOL_CALL_COUNT_METRIC, /*inc*/ 1, &[])?;
|
||||||
metrics.record_duration(TOOL_CALL_DURATION_METRIC, Duration::from_millis(25), &[])?;
|
metrics.record_duration(TOOL_CALL_DURATION_METRIC, Duration::from_millis(25), &[])?;
|
||||||
|
metrics.counter(TURN_COST_MICROUSD_METRIC, /*inc*/ 1, &[])?;
|
||||||
metrics.histogram(TURN_TOKEN_USAGE_METRIC, /*value*/ 100, &[])?;
|
metrics.histogram(TURN_TOKEN_USAGE_METRIC, /*value*/ 100, &[])?;
|
||||||
metrics.counter("codex.turns", /*inc*/ 1, &[])?;
|
metrics.counter("codex.turns", /*inc*/ 1, &[])?;
|
||||||
metrics.shutdown()?;
|
metrics.shutdown()?;
|
||||||
|
|||||||
@@ -155,3 +155,68 @@ fn manager_snapshot_metrics_collects_without_shutdown() -> Result<()> {
|
|||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn manager_turn_cost_records_microusd_metric() -> Result<()> {
|
||||||
|
let exporter = InMemoryMetricExporter::default();
|
||||||
|
let config = MetricsConfig::in_memory("test", "codex-cli", env!("CARGO_PKG_VERSION"), exporter)
|
||||||
|
.with_runtime_reader();
|
||||||
|
let metrics = MetricsClient::new(config)?;
|
||||||
|
let thread_id = ThreadId::new();
|
||||||
|
let conversation_id = thread_id.to_string();
|
||||||
|
let manager = SessionTelemetry::new(
|
||||||
|
thread_id,
|
||||||
|
"gpt-5.6",
|
||||||
|
"gpt-5.6",
|
||||||
|
/*account_id*/ None,
|
||||||
|
/*account_email*/ None,
|
||||||
|
Some(TelemetryAuthMode::ApiKey),
|
||||||
|
"test_originator".to_string(),
|
||||||
|
/*log_user_prompts*/ false,
|
||||||
|
"tty".to_string(),
|
||||||
|
SessionSource::Cli,
|
||||||
|
)
|
||||||
|
.with_metrics(metrics);
|
||||||
|
|
||||||
|
manager.record_turn_cost(
|
||||||
|
"turn-123",
|
||||||
|
"0.0001245",
|
||||||
|
/*interrupted*/ false,
|
||||||
|
Some("fast"),
|
||||||
|
Some("high"),
|
||||||
|
);
|
||||||
|
|
||||||
|
let snapshot = manager.snapshot_metrics()?;
|
||||||
|
let metric = find_metric(&snapshot, "codex.turn.cost_microusd")
|
||||||
|
.expect("turn-cost microdollar metric missing");
|
||||||
|
let point = match metric.data() {
|
||||||
|
AggregatedMetrics::U64(MetricData::Sum(sum)) => {
|
||||||
|
sum.data_points().next().expect("turn-cost data point")
|
||||||
|
}
|
||||||
|
_ => panic!("unexpected turn-cost metric data type"),
|
||||||
|
};
|
||||||
|
assert_eq!(point.value(), 125);
|
||||||
|
assert_eq!(
|
||||||
|
attributes_to_map(point.attributes()),
|
||||||
|
BTreeMap::from([
|
||||||
|
(
|
||||||
|
"app.version".to_string(),
|
||||||
|
env!("CARGO_PKG_VERSION").to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"auth_mode".to_string(),
|
||||||
|
TelemetryAuthMode::ApiKey.to_string(),
|
||||||
|
),
|
||||||
|
("conversation.id".to_string(), conversation_id),
|
||||||
|
("model".to_string(), "gpt-5.6".to_string()),
|
||||||
|
("originator".to_string(), "test_originator".to_string()),
|
||||||
|
("reasoning_effort".to_string(), "high".to_string()),
|
||||||
|
("session_source".to_string(), "cli".to_string()),
|
||||||
|
("speed".to_string(), "fast".to_string()),
|
||||||
|
("turn.id".to_string(), "turn-123".to_string()),
|
||||||
|
("turn.interrupted".to_string(), "false".to_string()),
|
||||||
|
])
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
Reference in new issue
Block a user