From 0a76a2520f45af68b7a5386688a9714af0a3ccd2 Mon Sep 17 00:00:00 2001 From: jif Date: Wed, 30 Sep 2026 17:28:58 +0000 Subject: [PATCH] Export skill invocation events through OpenTelemetry (#49689) ## What changed Emit `codex.skill_invocation` logs for explicit skill injection and detected implicit use, including skill-tool reads. Include skill name, invocation type, conversation, turn, model, and client metadata, plus scope, plugin ID, and ChatGPT user and account IDs when available. Exclude skill contents, resource paths, email addresses, prompts, and tool payloads. Use the existing OTEL log exporter without an additional opt-in; `otel.exporter = "none"` continues to export no logs. Document the event fields, exporter configuration, and detection semantics in `codex-rs/otel/README.md`. GitOrigin-RevId: 0f9c492c3f224455f3de3660be89692d776c1951 --- codex-rs/core/src/session/session.rs | 2 + codex-rs/core/src/skills.rs | 23 +++++++ codex-rs/ext/skills/src/tools/mod.rs | 21 +++++++ codex-rs/otel/README.md | 26 ++++++++ codex-rs/otel/src/events/session_telemetry.rs | 2 + codex-rs/otel/src/lib.rs | 3 + codex-rs/otel/src/skill_invocation.rs | 61 +++++++++++++++++++ 7 files changed, 138 insertions(+) create mode 100644 codex-rs/otel/src/skill_invocation.rs diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index b85caf0581..c012ae02c5 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -1303,6 +1303,7 @@ impl Session { session_configuration.session_source.clone(), ) .with_auth_env(auth_env_telemetry.to_otel_metadata()) + .with_user_id(telemetry_auth.and_then(CodexAuth::get_chatgpt_user_id)) .with_tool_result_log_config(config.otel.tool_result); if let Some(metrics) = thread_extension_data.get::() { session_telemetry = session_telemetry.with_metrics(metrics.as_ref().clone()); @@ -1665,6 +1666,7 @@ impl Session { let session_extension_data = codex_extension_api::ExtensionData::new(session_id.to_string()); session_extension_data.insert(analytics_events_client.clone()); + session_extension_data.insert(session_telemetry.clone()); let mcp_resource_client = Arc::new(McpResourceClient::new(Arc::clone(&mcp_runtime))); let extension_metrics = extension_metrics::from_session_telemetry(session_telemetry.clone()); diff --git a/codex-rs/core/src/skills.rs b/codex-rs/core/src/skills.rs index 6dd7938fb1..39cfdde78c 100644 --- a/codex-rs/core/src/skills.rs +++ b/codex-rs/core/src/skills.rs @@ -8,6 +8,8 @@ use codex_analytics::TrackEventsContext; use codex_analytics::build_track_events_context; use codex_extension_api::SkillInvocationInput; use codex_extension_api::SkillInvocationKind; +use codex_otel::SkillInvocationEvent; +use codex_otel::SkillInvocationType; use codex_otel::sanitize_metric_tag_value; use codex_protocol::protocol::SkillScope; use codex_skills::SkillMetadata; @@ -86,6 +88,15 @@ pub(crate) async fn emit_explicit_skill_invocations( { continue; } + turn_context + .session_telemetry + .skill_invocation(SkillInvocationEvent { + turn_id: &turn_context.sub_id, + skill_name: &skill.name, + scope: Some(skill.scope), + plugin_id: skill.plugin_id.as_deref(), + invocation_type: SkillInvocationType::Explicit, + }); for contributor in sess.services.extensions.skill_invocation_contributors() { contributor .on_skill_invocation(SkillInvocationInput { @@ -160,6 +171,18 @@ pub(crate) async fn maybe_emit_implicit_skill_invocation( if !inserted { return; } + turn_context + .session_telemetry + .skill_invocation(SkillInvocationEvent { + turn_id: &turn_context.sub_id, + skill_name: &skill_name, + scope: match &invocation.location { + SkillInvocationLocation::Host { scope, .. } => Some(*scope), + SkillInvocationLocation::Resource { scope, .. } => *scope, + }, + plugin_id: invocation.plugin_id.as_deref(), + invocation_type: SkillInvocationType::Implicit, + }); let skill_name_tag = sanitize_metric_tag_value(skill_name.as_str()); let plugin_id_tag = sanitize_metric_tag_value(invocation.plugin_id.as_deref().unwrap_or("unattributed")); diff --git a/codex-rs/ext/skills/src/tools/mod.rs b/codex-rs/ext/skills/src/tools/mod.rs index 167d35893e..e34730c480 100644 --- a/codex-rs/ext/skills/src/tools/mod.rs +++ b/codex-rs/ext/skills/src/tools/mod.rs @@ -25,6 +25,9 @@ use codex_extension_api::ToolSpec; use codex_extension_api::parse_tool_input_schema; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_mcp::McpResourceClient; +use codex_otel::SessionTelemetry; +use codex_otel::SkillInvocationEvent; +use codex_otel::SkillInvocationType; use codex_otel::sanitize_metric_tag_value; use codex_tools::ResponsesApiNamespace; use codex_tools::ResponsesApiNamespaceTool; @@ -99,6 +102,7 @@ pub(crate) fn skill_tools( #[derive(Clone)] pub(crate) struct SkillAnalytics { client: AnalyticsEventsClient, + telemetry: Option>, metrics: Option>, turn_metrics: Option>, thread_id: String, @@ -115,6 +119,7 @@ impl SkillAnalytics { Some(Self { client: client.as_ref().clone(), + telemetry: session_store.get::(), metrics: session_store .get::() .and_then(|state| state.extension_metrics.clone()), @@ -137,6 +142,22 @@ impl SkillAnalytics { turn_id: String, invocation_type: InvocationType, ) { + if let Some(telemetry) = &self.telemetry { + telemetry + .as_ref() + .clone() + .with_model(&model, &model) + .skill_invocation(SkillInvocationEvent { + turn_id: &turn_id, + skill_name: &skill.name, + scope: skill.analytics_scope, + plugin_id: skill.plugin_id.as_deref(), + invocation_type: match invocation_type { + InvocationType::Explicit => SkillInvocationType::Explicit, + InvocationType::Implicit => SkillInvocationType::Implicit, + }, + }); + } let turn_metrics = self .turn_metrics .as_ref() diff --git a/codex-rs/otel/README.md b/codex-rs/otel/README.md index 440d59535b..0de50826a2 100644 --- a/codex-rs/otel/README.md +++ b/codex-rs/otel/README.md @@ -96,6 +96,32 @@ let manager = SessionTelemetry::new( manager.user_prompt(&prompt_items); ``` +### Skill invocation logs + +`codex.skill_invocation` records explicit skill injection and detected implicit use, +including reads through the skills tools. It contains `skill.name`, +`skill.invocation_type` (`explicit` or `implicit`), `conversation.id`, `turn.id`, +model and client metadata, and available `skill.scope`, `skill.plugin_id`, +`user.id`, and `user.account_id`. The user ID identifies the authenticated ChatGPT +user; the account ID identifies the workspace. Missing identity fields are omitted, +including the user ID for API-key authentication. + +The event excludes skill contents, descriptions, resource paths, email addresses, +prompts, tool arguments, and tool output. It reports detected use, not successful +task completion. Shell-based implicit detection is best effort and is deduplicated +per skill within a turn; explicit injection and later implicit access are distinct +events. Skill-tool reads retain their existing invocation behavior. + +Skill events use the existing OTEL log exporter and are included automatically +when `otel.exporter` points to an OTLP HTTP or gRPC destination. The default +`otel.exporter = "none"` exports no logs. No additional configuration or skill-event +opt-in is required; the destination also receives the other existing log events. + +```toml +[otel] +exporter = { otlp-http = { endpoint = "https://collector.example.com/v1/logs", protocol = "json" } } +``` + ### Agent response logs Set `otel.log_agent_responses = true` with an OTLP HTTP or gRPC log exporter to diff --git a/codex-rs/otel/src/events/session_telemetry.rs b/codex-rs/otel/src/events/session_telemetry.rs index 33f2c82037..352265d9ea 100644 --- a/codex-rs/otel/src/events/session_telemetry.rs +++ b/codex-rs/otel/src/events/session_telemetry.rs @@ -100,6 +100,7 @@ pub struct SessionTelemetryMetadata { pub(crate) auth_mode: Option, pub(crate) auth_env: AuthEnvTelemetryMetadata, pub(crate) account_id: Option, + pub(crate) user_id: Option, pub(crate) account_email: Option, pub(crate) originator: String, pub(crate) product_sku: Option<&'static str>, @@ -587,6 +588,7 @@ impl SessionTelemetry { auth_mode: auth_mode.map(|m| m.to_string()), auth_env: AuthEnvTelemetryMetadata::default(), account_id, + user_id: None, account_email, originator: sanitize_metric_tag_value(originator.as_str()), product_sku: None, diff --git a/codex-rs/otel/src/lib.rs b/codex-rs/otel/src/lib.rs index 54e993a41a..690fdd9094 100644 --- a/codex-rs/otel/src/lib.rs +++ b/codex-rs/otel/src/lib.rs @@ -9,6 +9,7 @@ mod agent_response; mod guardian_assessment; mod network_policy; mod otlp; +mod skill_invocation; mod targets; mod tool_result; @@ -33,6 +34,8 @@ pub use crate::metrics::runtime_metrics::RuntimeMetricsSummary; pub use crate::metrics::timer::Timer; pub use crate::metrics::*; pub use crate::provider::OtelProvider; +pub use crate::skill_invocation::SkillInvocationEvent; +pub use crate::skill_invocation::SkillInvocationType; pub use crate::trace_context::context_from_w3c_trace_context; pub use crate::trace_context::current_span_trace_id; pub use crate::trace_context::current_span_w3c_trace_context; diff --git a/codex-rs/otel/src/skill_invocation.rs b/codex-rs/otel/src/skill_invocation.rs new file mode 100644 index 0000000000..1bc3b2a9e0 --- /dev/null +++ b/codex-rs/otel/src/skill_invocation.rs @@ -0,0 +1,61 @@ +//! Exports skill-use metadata without skill contents, filesystem paths, or tool payloads. +//! Detection belongs to the caller; an invocation does not imply successful task completion. + +use crate::SessionTelemetry; +use crate::events::shared::timestamp; +use crate::targets::OTEL_LOG_ONLY_TARGET; +use codex_protocol::protocol::SkillScope; + +/// How Codex detected the skill's use. +pub enum SkillInvocationType { + Explicit, + Implicit, +} + +/// Metadata for a detected skill invocation. Never include skill contents or resource paths. +pub struct SkillInvocationEvent<'a> { + pub turn_id: &'a str, + pub skill_name: &'a str, + pub scope: Option, + pub plugin_id: Option<&'a str>, + pub invocation_type: SkillInvocationType, +} + +impl SessionTelemetry { + pub fn with_user_id(mut self, user_id: Option) -> Self { + self.metadata.user_id = user_id; + self + } + + pub fn skill_invocation(&self, event: SkillInvocationEvent<'_>) { + let invocation_type = match event.invocation_type { + SkillInvocationType::Explicit => "explicit", + SkillInvocationType::Implicit => "implicit", + }; + let scope = event.scope.map(|scope| match scope { + SkillScope::User => "user", + SkillScope::Repo => "repo", + SkillScope::System => "system", + SkillScope::Admin => "admin", + }); + tracing::event!( + name: "codex.skill_invocation", + target: OTEL_LOG_ONLY_TARGET, + tracing::Level::INFO, + event.name = "codex.skill_invocation", + event.timestamp = %timestamp(), + conversation.id = %self.metadata.conversation_id, + turn.id = event.turn_id, + user.id = self.metadata.user_id.as_deref(), + user.account_id = self.metadata.account_id.as_deref(), + skill.name = event.skill_name, + skill.scope = scope, + skill.plugin_id = event.plugin_id, + skill.invocation_type = invocation_type, + model = %self.metadata.model, + app.version = self.metadata.app_version, + originator = %self.metadata.originator, + auth_mode = self.metadata.auth_mode.as_deref(), + ); + } +}