mirror of
https://github.com/openai/codex.git
synced 2026-10-20 00:18:49 +00:00
Track resource-backed skill invocations (#38066)
## What changed - Emit skill invocation analytics when a resource-backed skill is explicitly selected or its main resource is successfully read from the first page. - Preserve provider-supplied skill IDs and user scope when available, and derive a stable ID from the main resource otherwise. - Deduplicate repeated implicit reads of the same main resource within a turn. ## Testing - Extend the orchestrator skill integration test to cover explicit and implicit invocation events, stable fallback IDs, failed reads, and repeated-read deduplication. GitOrigin-RevId: 7d8453b31b5a2205847e65cda2c46183d6c97b45
This commit is contained in:
committed by
copyberry
parent
104e25ac5a
commit
c8f673fddc
Generated
+1
@@ -4156,6 +4156,7 @@ dependencies = [
|
||||
name = "codex-skills-extension"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"codex-analytics",
|
||||
"codex-config",
|
||||
"codex-exec-server",
|
||||
"codex-extension-api",
|
||||
|
||||
@@ -81,6 +81,7 @@ use crate::facts::PluginState;
|
||||
use crate::facts::PluginStateChangedInput;
|
||||
use crate::facts::PluginUsedInput;
|
||||
use crate::facts::SkillInvocation;
|
||||
use crate::facts::SkillInvocationLocation;
|
||||
use crate::facts::SkillInvokedInput;
|
||||
use crate::facts::SubAgentThreadStartedInput;
|
||||
use crate::facts::ThreadInitializationMode;
|
||||
@@ -3824,8 +3825,10 @@ async fn reducer_ingests_skill_invoked_fact() {
|
||||
tracking,
|
||||
invocations: vec![SkillInvocation {
|
||||
skill_name: "doc".to_string(),
|
||||
skill_scope: codex_protocol::protocol::SkillScope::User,
|
||||
skill_path,
|
||||
location: SkillInvocationLocation::Host {
|
||||
path: skill_path,
|
||||
scope: codex_protocol::protocol::SkillScope::User,
|
||||
},
|
||||
plugin_id: None,
|
||||
remote_plugin_id: None,
|
||||
invocation_type: InvocationType::Explicit,
|
||||
@@ -3871,8 +3874,10 @@ async fn reducer_includes_plugin_ids_for_plugin_skill_invocations() {
|
||||
tracking,
|
||||
invocations: vec![SkillInvocation {
|
||||
skill_name: "sample:doc".to_string(),
|
||||
skill_scope: codex_protocol::protocol::SkillScope::User,
|
||||
skill_path,
|
||||
location: SkillInvocationLocation::Host {
|
||||
path: skill_path,
|
||||
scope: codex_protocol::protocol::SkillScope::User,
|
||||
},
|
||||
plugin_id: Some("sample@test".to_string()),
|
||||
remote_plugin_id: Some("plugins~Plugin_sample".to_string()),
|
||||
invocation_type: InvocationType::Explicit,
|
||||
|
||||
@@ -328,13 +328,25 @@ impl From<InputError> for TurnSteerRejectionReason {
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SkillInvocation {
|
||||
pub skill_name: String,
|
||||
pub skill_scope: SkillScope,
|
||||
pub skill_path: PathBuf,
|
||||
pub location: SkillInvocationLocation,
|
||||
pub plugin_id: Option<String>,
|
||||
pub remote_plugin_id: Option<String>,
|
||||
pub invocation_type: InvocationType,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum SkillInvocationLocation {
|
||||
Host {
|
||||
path: PathBuf,
|
||||
scope: SkillScope,
|
||||
},
|
||||
Resource {
|
||||
id: String,
|
||||
skill_id: Option<String>,
|
||||
scope: Option<SkillScope>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Serialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum InvocationType {
|
||||
|
||||
@@ -54,6 +54,7 @@ pub use facts::PluginInstallRequested;
|
||||
pub use facts::PluginInstallRequestedPlugin;
|
||||
pub use facts::PluginInstallSource;
|
||||
pub use facts::SkillInvocation;
|
||||
pub use facts::SkillInvocationLocation;
|
||||
pub use facts::SubAgentThreadStartedInput;
|
||||
pub use facts::ThreadInitializationMode;
|
||||
pub use facts::TrackEventsContext;
|
||||
|
||||
@@ -86,11 +86,13 @@ use crate::facts::ExternalAgentConfigImportFailureInput;
|
||||
use crate::facts::HookRunInput;
|
||||
use crate::facts::ImagePreparationFact;
|
||||
use crate::facts::ImagePreparationMetadata;
|
||||
use crate::facts::InvocationType;
|
||||
use crate::facts::PluginInstallFailedInput;
|
||||
use crate::facts::PluginInstallRequestedInput;
|
||||
use crate::facts::PluginState;
|
||||
use crate::facts::PluginStateChangedInput;
|
||||
use crate::facts::PluginUsedInput;
|
||||
use crate::facts::SkillInvocationLocation;
|
||||
use crate::facts::SkillInvokedInput;
|
||||
use crate::facts::SubAgentThreadStartedInput;
|
||||
use crate::facts::ThreadInitializationMode;
|
||||
@@ -153,6 +155,7 @@ use codex_protocol::request_permissions::PermissionGrantScope as CorePermissionG
|
||||
use codex_protocol::request_permissions::RequestPermissionsResponse as CoreRequestPermissionsResponse;
|
||||
use sha1::Digest;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::collections::VecDeque;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
@@ -394,6 +397,7 @@ struct TurnState {
|
||||
latest_diff: Option<String>,
|
||||
steer_count: usize,
|
||||
tool_counts: TurnToolCounts,
|
||||
resource_skill_invocations: HashSet<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Hash, Eq, PartialEq)]
|
||||
@@ -1078,26 +1082,54 @@ impl AnalyticsReducer {
|
||||
invocations,
|
||||
} = input;
|
||||
for invocation in invocations {
|
||||
let skill_scope = match invocation.skill_scope {
|
||||
SkillScope::User => "user",
|
||||
SkillScope::Repo => "repo",
|
||||
SkillScope::System => "system",
|
||||
SkillScope::Admin => "admin",
|
||||
let (skill_id, repo_url, skill_scope) = match invocation.location {
|
||||
SkillInvocationLocation::Host { path, scope } => {
|
||||
let skill_scope = match scope {
|
||||
SkillScope::User => "user",
|
||||
SkillScope::Repo => "repo",
|
||||
SkillScope::System => "system",
|
||||
SkillScope::Admin => "admin",
|
||||
};
|
||||
let repo_root = get_git_repo_root(path.as_path());
|
||||
let repo_url = if let Some(root) = repo_root.as_ref() {
|
||||
collect_git_info(root)
|
||||
.await
|
||||
.and_then(|info| info.repository_url)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let skill_id = skill_id_for_local_skill(
|
||||
repo_url.as_deref(),
|
||||
repo_root.as_deref(),
|
||||
path.as_path(),
|
||||
invocation.skill_name.as_str(),
|
||||
);
|
||||
(skill_id, repo_url, Some(skill_scope.to_string()))
|
||||
}
|
||||
SkillInvocationLocation::Resource {
|
||||
id,
|
||||
skill_id,
|
||||
scope,
|
||||
} => {
|
||||
if matches!(invocation.invocation_type, InvocationType::Implicit) {
|
||||
let turn_state = self.turns.entry(tracking.turn_id.clone()).or_default();
|
||||
if !turn_state.resource_skill_invocations.insert(id.clone()) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let skill_id = skill_id
|
||||
.unwrap_or_else(|| format!("{:x}", sha1::Sha1::digest(id.as_bytes())));
|
||||
let skill_scope = scope
|
||||
.map(|scope| match scope {
|
||||
SkillScope::User => "user",
|
||||
SkillScope::Repo => "repo",
|
||||
SkillScope::System => "system",
|
||||
SkillScope::Admin => "admin",
|
||||
})
|
||||
.map(str::to_owned);
|
||||
(skill_id, None, skill_scope)
|
||||
}
|
||||
};
|
||||
let repo_root = get_git_repo_root(invocation.skill_path.as_path());
|
||||
let repo_url = if let Some(root) = repo_root.as_ref() {
|
||||
collect_git_info(root)
|
||||
.await
|
||||
.and_then(|info| info.repository_url)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let skill_id = skill_id_for_local_skill(
|
||||
repo_url.as_deref(),
|
||||
repo_root.as_deref(),
|
||||
invocation.skill_path.as_path(),
|
||||
invocation.skill_name.as_str(),
|
||||
);
|
||||
out.push(TrackEventRequest::SkillInvocation(
|
||||
SkillInvocationEventRequest {
|
||||
event_type: "skill_invocation",
|
||||
@@ -1110,7 +1142,7 @@ impl AnalyticsReducer {
|
||||
model_slug: Some(tracking.model_slug.clone()),
|
||||
product_client_id: Some(tracking.product_client_id.clone()),
|
||||
repo_url,
|
||||
skill_scope: Some(skill_scope.to_string()),
|
||||
skill_scope,
|
||||
plugin_id: invocation.plugin_id,
|
||||
remote_plugin_id: invocation.remote_plugin_id,
|
||||
},
|
||||
|
||||
@@ -1186,6 +1186,7 @@ impl Session {
|
||||
));
|
||||
let session_extension_data =
|
||||
codex_extension_api::ExtensionData::new(session_id.to_string());
|
||||
session_extension_data.insert(analytics_events_client.clone());
|
||||
let mcp_resource_client = Arc::new(McpResourceClient::new(Arc::clone(&mcp_runtime)));
|
||||
let extension_metrics =
|
||||
extension_metrics::from_session_telemetry(session_telemetry.clone());
|
||||
|
||||
+18
-13
@@ -3,6 +3,7 @@ use crate::session::session::Session;
|
||||
use crate::session::turn_context::TurnContext;
|
||||
use codex_analytics::InvocationType;
|
||||
use codex_analytics::SkillInvocation;
|
||||
use codex_analytics::SkillInvocationLocation;
|
||||
use codex_analytics::TrackEventsContext;
|
||||
use codex_analytics::build_track_events_context;
|
||||
use codex_extension_api::SkillInvocationInput;
|
||||
@@ -64,8 +65,10 @@ pub(crate) fn emit_explicit_skill_invocations(
|
||||
.iter()
|
||||
.map(|skill| SkillInvocation {
|
||||
skill_name: skill.name.clone(),
|
||||
skill_scope: skill.scope,
|
||||
skill_path: skill.path_to_skills_md.to_path_buf(),
|
||||
location: SkillInvocationLocation::Host {
|
||||
path: skill.path_to_skills_md.to_path_buf(),
|
||||
scope: skill.scope,
|
||||
},
|
||||
plugin_id: skill.plugin_id.clone(),
|
||||
remote_plugin_id: skill.remote_plugin_id.clone(),
|
||||
invocation_type: InvocationType::Explicit,
|
||||
@@ -89,21 +92,23 @@ pub(crate) async fn maybe_emit_implicit_skill_invocation(
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
let invocation = SkillInvocation {
|
||||
skill_name: candidate.name,
|
||||
skill_scope: candidate.scope,
|
||||
skill_path: candidate.path_to_skills_md.to_path_buf(),
|
||||
plugin_id: candidate.plugin_id,
|
||||
remote_plugin_id: candidate.remote_plugin_id,
|
||||
invocation_type: InvocationType::Implicit,
|
||||
};
|
||||
let skill_scope = match invocation.skill_scope {
|
||||
let skill_scope = match candidate.scope {
|
||||
SkillScope::User => "user",
|
||||
SkillScope::Repo => "repo",
|
||||
SkillScope::System => "system",
|
||||
SkillScope::Admin => "admin",
|
||||
};
|
||||
let skill_path = invocation.skill_path.to_string_lossy();
|
||||
let skill_path = candidate.path_to_skills_md.to_string_lossy().into_owned();
|
||||
let invocation = SkillInvocation {
|
||||
skill_name: candidate.name,
|
||||
location: SkillInvocationLocation::Host {
|
||||
path: candidate.path_to_skills_md.to_path_buf(),
|
||||
scope: candidate.scope,
|
||||
},
|
||||
plugin_id: candidate.plugin_id,
|
||||
remote_plugin_id: candidate.remote_plugin_id,
|
||||
invocation_type: InvocationType::Implicit,
|
||||
};
|
||||
let skill_name = invocation.skill_name.clone();
|
||||
let seen_key = format!("{skill_scope}:{skill_path}:{skill_name}");
|
||||
let inserted = {
|
||||
@@ -125,7 +130,7 @@ pub(crate) async fn maybe_emit_implicit_skill_invocation(
|
||||
thread_store: &sess.services.thread_extension_data,
|
||||
turn_store: turn_context.extension_data.as_ref(),
|
||||
turn_id: turn_context.sub_id.as_str(),
|
||||
skill_resource: skill_path.as_ref(),
|
||||
skill_resource: skill_path.as_str(),
|
||||
kind: SkillInvocationKind::Implicit,
|
||||
})
|
||||
.await;
|
||||
|
||||
@@ -57,11 +57,13 @@ use core_test_support::wait_for_mcp_server;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::Value;
|
||||
use serde_json::json;
|
||||
use sha1::Digest;
|
||||
use tempfile::TempDir;
|
||||
use tokio::time::Duration;
|
||||
use tokio::time::Instant;
|
||||
use toml::toml;
|
||||
use wiremock::Mock;
|
||||
use wiremock::MockServer;
|
||||
use wiremock::Request;
|
||||
use wiremock::ResponseTemplate;
|
||||
use wiremock::matchers::method;
|
||||
@@ -262,6 +264,34 @@ fn catalog_extensions(
|
||||
(Arc::new(extensions.build()), event_rx)
|
||||
}
|
||||
|
||||
async fn wait_for_analytics_events(
|
||||
server: &MockServer,
|
||||
event_type: &str,
|
||||
expected_count: usize,
|
||||
) -> Vec<Value> {
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
loop {
|
||||
let events = server
|
||||
.received_requests()
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.filter(|request| request.url.path() == "/codex/analytics-events/events")
|
||||
.filter_map(|request| serde_json::from_slice::<Value>(&request.body).ok())
|
||||
.flat_map(|payload| payload["events"].as_array().cloned().unwrap_or_default())
|
||||
.filter(|event| event["event_type"] == event_type)
|
||||
.collect::<Vec<_>>();
|
||||
if events.len() >= expected_count {
|
||||
return events;
|
||||
}
|
||||
assert!(
|
||||
Instant::now() < deadline,
|
||||
"timed out waiting for {event_type} analytics"
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
}
|
||||
|
||||
fn configure_catalog_test(config: &mut Config) {
|
||||
config.include_skill_instructions = true;
|
||||
config
|
||||
@@ -815,6 +845,10 @@ async fn explicit_only_orchestrator_skill_is_hidden_but_can_be_invoked() -> Resu
|
||||
const REFERENCED_RESOURCE: &str = "skill://demo/explicit-only/references/guide.md";
|
||||
const REFERENCED_CONTENTS: &str = "# Referenced guide";
|
||||
const READ_CALL_ID: &str = "read-explicit-only-resource";
|
||||
const MAIN_READ_CALL_ID: &str = "read-explicit-only-main";
|
||||
const REPEATED_MAIN_READ_CALL_ID: &str = "read-explicit-only-main-again";
|
||||
const INVALID_CURSOR_CALL_ID: &str = "read-explicit-only-invalid-cursor";
|
||||
const MISSING_PACKAGE_CALL_ID: &str = "read-missing-package";
|
||||
|
||||
let server = responses::start_mock_server().await;
|
||||
let apps_server = AppsTestServer::mount(&server).await?;
|
||||
@@ -839,6 +873,35 @@ async fn explicit_only_orchestrator_skill_is_hidden_but_can_be_invoked() -> Resu
|
||||
ev_completed("resp-1"),
|
||||
]),
|
||||
sse(vec![ev_response_created("resp-2"), ev_completed("resp-2")]),
|
||||
sse(vec![
|
||||
ev_response_created("resp-3"),
|
||||
responses::ev_function_call_with_namespace(
|
||||
MAIN_READ_CALL_ID,
|
||||
"skills",
|
||||
"read",
|
||||
&json!({ "package": SKILL_PACKAGE }).to_string(),
|
||||
),
|
||||
responses::ev_function_call_with_namespace(
|
||||
REPEATED_MAIN_READ_CALL_ID,
|
||||
"skills",
|
||||
"read",
|
||||
&json!({ "package": SKILL_PACKAGE, "resource": MAIN_RESOURCE }).to_string(),
|
||||
),
|
||||
responses::ev_function_call_with_namespace(
|
||||
INVALID_CURSOR_CALL_ID,
|
||||
"skills",
|
||||
"read",
|
||||
&json!({ "package": SKILL_PACKAGE, "cursor": "invalid" }).to_string(),
|
||||
),
|
||||
responses::ev_function_call_with_namespace(
|
||||
MISSING_PACKAGE_CALL_ID,
|
||||
"skills",
|
||||
"read",
|
||||
&json!({ "package": "skill://demo/missing" }).to_string(),
|
||||
),
|
||||
ev_completed("resp-3"),
|
||||
]),
|
||||
sse(vec![ev_response_created("resp-4"), ev_completed("resp-4")]),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
@@ -990,6 +1053,39 @@ async fn explicit_only_orchestrator_skill_is_hidden_but_can_be_invoked() -> Resu
|
||||
"next_cursor": null,
|
||||
})
|
||||
);
|
||||
let events = wait_for_analytics_events(&server, "skill_invocation", /*expected_count*/ 1).await;
|
||||
assert_eq!(events.len(), 1);
|
||||
assert_eq!(events[0]["skill_name"], "demo:explicit-only");
|
||||
assert_eq!(events[0]["event_params"]["invoke_type"], "explicit");
|
||||
|
||||
test.submit_turn("Continue without explicitly selecting a skill.")
|
||||
.await?;
|
||||
let requests = response.requests();
|
||||
assert_eq!(requests.len(), 4);
|
||||
for call_id in [MAIN_READ_CALL_ID, REPEATED_MAIN_READ_CALL_ID] {
|
||||
let output = requests[3]
|
||||
.function_call_output_text(call_id)
|
||||
.expect("skills.read should return the main resource");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Value>(&output)?["resource"],
|
||||
MAIN_RESOURCE
|
||||
);
|
||||
}
|
||||
for call_id in [INVALID_CURSOR_CALL_ID, MISSING_PACKAGE_CALL_ID] {
|
||||
assert!(
|
||||
requests[3].function_call_output_text(call_id).is_some(),
|
||||
"failed skills.read should return a tool error for {call_id}"
|
||||
);
|
||||
}
|
||||
|
||||
let events = wait_for_analytics_events(&server, "skill_invocation", /*expected_count*/ 2).await;
|
||||
assert_eq!(events.len(), 2, "repeated main reads must be deduplicated");
|
||||
assert_eq!(events[1]["skill_name"], "demo:explicit-only");
|
||||
assert_eq!(
|
||||
events[1]["skill_id"],
|
||||
format!("{:x}", sha1::Sha1::digest(MAIN_RESOURCE.as_bytes()))
|
||||
);
|
||||
assert_eq!(events[1]["event_params"]["invoke_type"], "implicit");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1635,29 +1731,9 @@ async fn production_turn_uses_provider_host_catalog_and_core_snapshot_injection(
|
||||
let user_text = request.message_input_texts("user").join("\n");
|
||||
assert!(user_text.contains(&snapshot_contents));
|
||||
assert!(!user_text.contains(provider_contents));
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
let app_mentioned_event = loop {
|
||||
let requests = server.received_requests().await.unwrap_or_default();
|
||||
if let Some(event) = requests
|
||||
.into_iter()
|
||||
.filter(|request| request.url.path() == "/codex/analytics-events/events")
|
||||
.find_map(|request| {
|
||||
let payload: serde_json::Value = serde_json::from_slice(&request.body).ok()?;
|
||||
payload["events"].as_array().and_then(|events| {
|
||||
events
|
||||
.iter()
|
||||
.find(|event| event["event_type"] == "codex_app_mentioned")
|
||||
.cloned()
|
||||
})
|
||||
})
|
||||
{
|
||||
break event;
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
panic!("timed out waiting for app mentioned analytics");
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
};
|
||||
let app_mentioned_events =
|
||||
wait_for_analytics_events(&server, "codex_app_mentioned", /*expected_count*/ 1).await;
|
||||
let app_mentioned_event = &app_mentioned_events[0];
|
||||
assert_eq!(
|
||||
app_mentioned_event["event_params"]["connector_id"],
|
||||
"calendar"
|
||||
|
||||
@@ -13,6 +13,7 @@ doctest = false
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
codex-analytics = { workspace = true }
|
||||
codex-config = { workspace = true }
|
||||
codex-exec-server = { workspace = true }
|
||||
codex-extension-api = { workspace = true }
|
||||
|
||||
@@ -178,6 +178,8 @@ pub struct SkillCatalogEntry {
|
||||
pub short_description: Option<String>,
|
||||
pub main_prompt: SkillResourceId,
|
||||
pub display_path: Option<String>,
|
||||
pub(crate) canonical_skill_id: Option<String>,
|
||||
pub(crate) analytics_scope: Option<SkillScope>,
|
||||
alias_root: Option<String>,
|
||||
alias_root_order: Option<usize>,
|
||||
prompt_scope: Option<SkillScope>,
|
||||
@@ -202,6 +204,8 @@ impl SkillCatalogEntry {
|
||||
short_description: None,
|
||||
main_prompt,
|
||||
display_path: None,
|
||||
canonical_skill_id: None,
|
||||
analytics_scope: None,
|
||||
alias_root: None,
|
||||
alias_root_order: None,
|
||||
prompt_scope: None,
|
||||
|
||||
@@ -3,6 +3,7 @@ use std::sync::Arc;
|
||||
|
||||
use crate::HostSkillsSnapshot;
|
||||
use crate::InjectedHostSkillPrompts;
|
||||
use codex_analytics::InvocationType;
|
||||
use codex_exec_server::ExecutorCapabilityDiscoverySnapshot;
|
||||
use codex_exec_server::FileSystemSandboxContext;
|
||||
use codex_exec_server::LOCAL_ENVIRONMENT_ID;
|
||||
@@ -65,6 +66,7 @@ use crate::state::HostSkillsStepState;
|
||||
use crate::state::SkillsSessionState;
|
||||
use crate::state::SkillsThreadState;
|
||||
use crate::state::SkillsTurnState;
|
||||
use crate::tools::SkillAnalytics;
|
||||
use crate::tools::SkillToolAuthority;
|
||||
use crate::tools::skill_tools;
|
||||
use crate::warnings::bounded_warnings;
|
||||
@@ -437,6 +439,7 @@ where
|
||||
let mut warnings = catalog.warnings.clone();
|
||||
let mut main_prompts_injected = false;
|
||||
let mut injected_host_skill_prompts = InjectedHostSkillPrompts::default();
|
||||
let analytics = SkillAnalytics::from_stores(session_store, thread_store);
|
||||
for entry in &selected_entries {
|
||||
match self
|
||||
.read_main_prompt(
|
||||
@@ -483,6 +486,15 @@ where
|
||||
main_prompts_injected = true;
|
||||
if entry.authority.kind == SkillSourceKind::Host {
|
||||
injected_host_skill_prompts.insert_path(entry.main_prompt.as_str());
|
||||
} else if let Some(analytics) = analytics.as_ref()
|
||||
&& let Some(model_info) = thread_store.get::<ModelInfo>()
|
||||
{
|
||||
analytics.track_skill_invocation(
|
||||
entry,
|
||||
model_info.slug.clone(),
|
||||
input.turn_id.clone(),
|
||||
InvocationType::Explicit,
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(message) => {
|
||||
@@ -548,10 +560,8 @@ impl<C> SkillsExtension<C> {
|
||||
|
||||
skill_tools(
|
||||
self.providers.clone(),
|
||||
session_store
|
||||
.get::<SkillsSessionState>()
|
||||
.and_then(|state| state.mcp_resources.clone()),
|
||||
thread_state,
|
||||
session_store,
|
||||
thread_store,
|
||||
orchestrator_available,
|
||||
executor_query,
|
||||
sandbox_contexts,
|
||||
|
||||
@@ -4,6 +4,7 @@ use std::time::Duration;
|
||||
use codex_mcp::CODEX_APPS_MCP_SERVER_NAME;
|
||||
use codex_protocol::mcp::Resource;
|
||||
use codex_protocol::mcp::ResourceContent;
|
||||
use codex_protocol::protocol::SkillScope;
|
||||
use url::Url;
|
||||
|
||||
use crate::catalog::SkillAuthority;
|
||||
@@ -253,7 +254,8 @@ fn catalog_entry_from_resource(resource: &Resource) -> Option<SkillCatalogEntry>
|
||||
.and_then(serde_json::Value::as_bool)
|
||||
.unwrap_or(true);
|
||||
let skill_name = normalized_label(meta.get("skill_name")?.as_str()?, MAX_SKILL_NAME_CHARS)?;
|
||||
let name = if meta.get("source").and_then(|value| value.as_str()) == Some("user") {
|
||||
let user_owned = meta.get("source").and_then(|value| value.as_str()) == Some("user");
|
||||
let name = if user_owned {
|
||||
skill_name
|
||||
} else {
|
||||
let plugin_name =
|
||||
@@ -265,7 +267,7 @@ fn catalog_entry_from_resource(resource: &Resource) -> Option<SkillCatalogEntry>
|
||||
let description = normalized_description(resource.description.as_deref().unwrap_or_default())?;
|
||||
let main_prompt = main_prompt_uri(uri);
|
||||
|
||||
let entry = SkillCatalogEntry::new(
|
||||
let mut entry = SkillCatalogEntry::new(
|
||||
SkillPackageId(uri.to_string()),
|
||||
SkillAuthority::new(SkillSourceKind::Orchestrator, CODEX_APPS_MCP_SERVER_NAME),
|
||||
name,
|
||||
@@ -274,6 +276,11 @@ fn catalog_entry_from_resource(resource: &Resource) -> Option<SkillCatalogEntry>
|
||||
)
|
||||
.with_display_path(uri)
|
||||
.with_alias_root(format!("skill://{namespace}"));
|
||||
entry.canonical_skill_id = meta
|
||||
.get("skill_id")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(str::to_owned);
|
||||
entry.analytics_scope = user_owned.then_some(SkillScope::User);
|
||||
|
||||
Some(if allow_implicit_invocation {
|
||||
entry
|
||||
|
||||
@@ -4,10 +4,17 @@ use std::hash::Hash;
|
||||
use std::hash::Hasher;
|
||||
use std::sync::Arc;
|
||||
|
||||
use codex_analytics::AnalyticsEventsClient;
|
||||
use codex_analytics::InvocationType;
|
||||
use codex_analytics::SkillInvocation;
|
||||
use codex_analytics::SkillInvocationLocation;
|
||||
use codex_analytics::build_track_events_context;
|
||||
use codex_exec_server::FileSystemSandboxContext;
|
||||
use codex_extension_api::ExtensionData;
|
||||
use codex_extension_api::FunctionCallError;
|
||||
use codex_extension_api::JsonToolOutput;
|
||||
use codex_extension_api::ResponsesApiTool;
|
||||
use codex_extension_api::ThreadOriginator;
|
||||
use codex_extension_api::ToolCall;
|
||||
use codex_extension_api::ToolExecutor;
|
||||
use codex_extension_api::ToolName;
|
||||
@@ -27,10 +34,12 @@ use tokio::sync::OnceCell;
|
||||
|
||||
use crate::catalog::SkillAuthority;
|
||||
use crate::catalog::SkillCatalog;
|
||||
use crate::catalog::SkillCatalogEntry;
|
||||
use crate::catalog::SkillSourceKind;
|
||||
use crate::provider::SkillListQuery;
|
||||
use crate::shadow_selection_experiment::ShadowSelectionExperiment;
|
||||
use crate::sources::SkillProviders;
|
||||
use crate::state::SkillsSessionState;
|
||||
use crate::state::SkillsThreadState;
|
||||
|
||||
mod list;
|
||||
@@ -42,17 +51,25 @@ const MAX_HANDLE_BYTES: usize = 2_048;
|
||||
|
||||
pub(crate) fn skill_tools(
|
||||
providers: SkillProviders,
|
||||
mcp_resources: Option<Arc<McpResourceClient>>,
|
||||
thread_state: Arc<SkillsThreadState>,
|
||||
session_store: &ExtensionData,
|
||||
thread_store: &ExtensionData,
|
||||
orchestrator_available: bool,
|
||||
executor_query: Option<SkillListQuery>,
|
||||
sandbox_contexts: Option<Arc<HashMap<String, FileSystemSandboxContext>>>,
|
||||
shadow_selection: Arc<ShadowSelectionExperiment>,
|
||||
) -> Vec<Arc<dyn ToolExecutor<ToolCall>>> {
|
||||
let Some(thread_state) = thread_store.get::<SkillsThreadState>() else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mcp_resources = session_store
|
||||
.get::<SkillsSessionState>()
|
||||
.and_then(|state| state.mcp_resources.clone());
|
||||
let analytics = SkillAnalytics::from_stores(session_store, thread_store);
|
||||
let context = SkillToolContext {
|
||||
providers,
|
||||
mcp_resources,
|
||||
thread_state,
|
||||
analytics,
|
||||
orchestrator_available,
|
||||
executor_query,
|
||||
sandbox_contexts,
|
||||
@@ -67,11 +84,64 @@ pub(crate) fn skill_tools(
|
||||
]
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct SkillAnalytics {
|
||||
client: AnalyticsEventsClient,
|
||||
thread_id: String,
|
||||
product_client_id: String,
|
||||
}
|
||||
|
||||
impl SkillAnalytics {
|
||||
pub(crate) fn from_stores(
|
||||
session_store: &ExtensionData,
|
||||
thread_store: &ExtensionData,
|
||||
) -> Option<Self> {
|
||||
let client = session_store.get::<AnalyticsEventsClient>()?;
|
||||
let originator = thread_store.get::<ThreadOriginator>()?;
|
||||
|
||||
Some(Self {
|
||||
client: client.as_ref().clone(),
|
||||
thread_id: thread_store.level_id().to_string(),
|
||||
product_client_id: originator.0.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn track_skill_invocation(
|
||||
&self,
|
||||
skill: &SkillCatalogEntry,
|
||||
model: String,
|
||||
turn_id: String,
|
||||
invocation_type: InvocationType,
|
||||
) {
|
||||
self.client.track_skill_invocations(
|
||||
build_track_events_context(
|
||||
model,
|
||||
self.thread_id.clone(),
|
||||
turn_id,
|
||||
self.product_client_id.clone(),
|
||||
),
|
||||
vec![SkillInvocation {
|
||||
skill_name: skill.name.clone(),
|
||||
location: SkillInvocationLocation::Resource {
|
||||
id: skill.main_prompt.as_str().to_string(),
|
||||
skill_id: skill.canonical_skill_id.clone(),
|
||||
scope: skill.analytics_scope,
|
||||
},
|
||||
// TODO: Include plugin identifiers once skills can be attributed to their plugin.
|
||||
plugin_id: None,
|
||||
remote_plugin_id: None,
|
||||
invocation_type,
|
||||
}],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct SkillToolContext {
|
||||
providers: SkillProviders,
|
||||
mcp_resources: Option<Arc<McpResourceClient>>,
|
||||
thread_state: Arc<SkillsThreadState>,
|
||||
analytics: Option<SkillAnalytics>,
|
||||
orchestrator_available: bool,
|
||||
executor_query: Option<SkillListQuery>,
|
||||
sandbox_contexts: Option<Arc<HashMap<String, FileSystemSandboxContext>>>,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use codex_analytics::InvocationType;
|
||||
use codex_extension_api::FunctionCallError;
|
||||
use codex_extension_api::ToolCall;
|
||||
use codex_extension_api::ToolExecutor;
|
||||
@@ -177,7 +178,21 @@ impl ToolExecutor<ToolCall> for ReadTool {
|
||||
));
|
||||
}
|
||||
let response = page_response(result.resource.as_str(), &result.contents, start)?;
|
||||
skill_json_output(&response, output_authority)
|
||||
let output = skill_json_output(&response, output_authority)?;
|
||||
|
||||
if requested_resource == main_prompt
|
||||
&& args.cursor.is_none()
|
||||
&& let Some(analytics) = self.context.analytics.as_ref()
|
||||
{
|
||||
analytics.track_skill_invocation(
|
||||
&skill_entry,
|
||||
call.model.clone(),
|
||||
call.turn_id.clone(),
|
||||
InvocationType::Implicit,
|
||||
);
|
||||
}
|
||||
|
||||
Ok(output)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user