mirror of
https://github.com/openai/codex.git
synced 2026-11-13 14:46:14 +00:00
Align shadow skill selection with observable sources (#32768)
## Why Shadow selection can observe invocations from host and orchestrator skills, but including executor skills in its candidates can skew the resulting metrics. ## What changed Limit eligible shadow-selection candidates to enabled, prompt-visible skills from host or orchestrator sources. ## Testing Extend the implicit-invocation test to add matching executor candidates and verify that the host skill remains the selected invocation hit. GitOrigin-RevId: fd444fe27254b5a880ef5c03c29b5e195127cd97
This commit is contained in:
1 parent
c100109280
commit
2b0b37abb7
2 files changed
+69
-3
No files matched your search
@@ -10,6 +10,7 @@ use codex_otel::MetricsClient;
|
|||||||
use codex_protocol::user_input::UserInput;
|
use codex_protocol::user_input::UserInput;
|
||||||
|
|
||||||
use crate::catalog::SkillCatalog;
|
use crate::catalog::SkillCatalog;
|
||||||
|
use crate::catalog::SkillSourceKind;
|
||||||
use crate::dynamic_skill_selector::CheapSkillSelection;
|
use crate::dynamic_skill_selector::CheapSkillSelection;
|
||||||
use crate::dynamic_skill_selector::CheapSkillSelector;
|
use crate::dynamic_skill_selector::CheapSkillSelector;
|
||||||
use crate::dynamic_skill_selector::SkillSelectionDocument;
|
use crate::dynamic_skill_selector::SkillSelectionDocument;
|
||||||
@@ -50,7 +51,16 @@ impl ShadowSelectionExperiment {
|
|||||||
.entries
|
.entries
|
||||||
.iter()
|
.iter()
|
||||||
.enumerate()
|
.enumerate()
|
||||||
.filter(|(_, entry)| entry.enabled && entry.prompt_visible)
|
.filter(|(_, entry)| {
|
||||||
|
entry.enabled
|
||||||
|
&& entry.prompt_visible
|
||||||
|
// Invocation observation currently exists only for host shell use and
|
||||||
|
// orchestrator reads. Keep the candidate set aligned with that universe.
|
||||||
|
&& matches!(
|
||||||
|
&entry.authority.kind,
|
||||||
|
SkillSourceKind::Host | SkillSourceKind::Orchestrator
|
||||||
|
)
|
||||||
|
})
|
||||||
.map(|(id, entry)| SkillSelectionDocument {
|
.map(|(id, entry)| SkillSelectionDocument {
|
||||||
id,
|
id,
|
||||||
name: entry.name.as_str(),
|
name: entry.name.as_str(),
|
||||||
|
|||||||
@@ -15,9 +15,12 @@ use codex_extension_api::ThreadStartInput;
|
|||||||
use codex_extension_api::ToolCall;
|
use codex_extension_api::ToolCall;
|
||||||
use codex_extension_api::ToolPayload;
|
use codex_extension_api::ToolPayload;
|
||||||
use codex_extension_api::TurnInputContext;
|
use codex_extension_api::TurnInputContext;
|
||||||
|
use codex_extension_api::WorldStateContributionInput;
|
||||||
use codex_mcp::CODEX_APPS_MCP_SERVER_NAME;
|
use codex_mcp::CODEX_APPS_MCP_SERVER_NAME;
|
||||||
use codex_otel::MetricsClient;
|
use codex_otel::MetricsClient;
|
||||||
use codex_otel::MetricsConfig;
|
use codex_otel::MetricsConfig;
|
||||||
|
use codex_protocol::capabilities::CapabilityRootLocation;
|
||||||
|
use codex_protocol::capabilities::SelectedCapabilityRoot;
|
||||||
use codex_protocol::protocol::SessionSource;
|
use codex_protocol::protocol::SessionSource;
|
||||||
use codex_protocol::protocol::SkillScope;
|
use codex_protocol::protocol::SkillScope;
|
||||||
use codex_protocol::protocol::TruncationPolicy;
|
use codex_protocol::protocol::TruncationPolicy;
|
||||||
@@ -39,6 +42,7 @@ use codex_skills_extension::provider::SkillProviderFuture;
|
|||||||
use codex_skills_extension::provider::SkillReadRequest;
|
use codex_skills_extension::provider::SkillReadRequest;
|
||||||
use codex_skills_extension::provider::SkillSearchRequest;
|
use codex_skills_extension::provider::SkillSearchRequest;
|
||||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||||
|
use codex_utils_path_uri::PathUri;
|
||||||
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
|
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
|
||||||
use opentelemetry_sdk::metrics::data::AggregatedMetrics;
|
use opentelemetry_sdk::metrics::data::AggregatedMetrics;
|
||||||
use opentelemetry_sdk::metrics::data::MetricData;
|
use opentelemetry_sdk::metrics::data::MetricData;
|
||||||
@@ -80,6 +84,40 @@ impl SkillProvider for OrchestratorProvider {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct ExecutorProvider;
|
||||||
|
|
||||||
|
impl SkillProvider for ExecutorProvider {
|
||||||
|
fn list(&self, _query: SkillListQuery) -> SkillProviderFuture<'_, SkillCatalog> {
|
||||||
|
Box::pin(std::future::ready(Ok(SkillCatalog {
|
||||||
|
entries: (0..5)
|
||||||
|
.map(|index| {
|
||||||
|
let resource = format!("skill://executor/demo-{index}/SKILL.md");
|
||||||
|
SkillCatalogEntry::new(
|
||||||
|
SkillPackageId(format!("executor/demo-{index}")),
|
||||||
|
SkillAuthority::new(SkillSourceKind::Executor, "executor"),
|
||||||
|
"演示文稿",
|
||||||
|
"创建演示文稿。",
|
||||||
|
SkillResourceId::new(resource),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
warnings: Vec::new(),
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read(&self, request: SkillReadRequest) -> SkillProviderFuture<'_, SkillReadResult> {
|
||||||
|
Box::pin(std::future::ready(Ok(SkillReadResult {
|
||||||
|
resource: request.resource,
|
||||||
|
contents: "# 演示文稿".to_string(),
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn search(&self, _request: SkillSearchRequest) -> SkillProviderFuture<'_, SkillSearchResult> {
|
||||||
|
Box::pin(std::future::ready(Ok(SkillSearchResult::default())))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn implicit_core_and_native_read_invocations_share_turn_local_recording() -> TestResult {
|
async fn implicit_core_and_native_read_invocations_share_turn_local_recording() -> TestResult {
|
||||||
let metrics = MetricsClient::new(
|
let metrics = MetricsClient::new(
|
||||||
@@ -174,7 +212,7 @@ async fn implicit_core_and_native_read_invocations_share_turn_local_recording()
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn host_snapshot_is_available_only_to_shadow_selection() -> TestResult {
|
async fn shadow_selection_uses_host_snapshot_and_excludes_executor_candidates() -> TestResult {
|
||||||
let metrics = MetricsClient::new(
|
let metrics = MetricsClient::new(
|
||||||
MetricsConfig::in_memory(
|
MetricsConfig::in_memory(
|
||||||
"test",
|
"test",
|
||||||
@@ -187,7 +225,7 @@ async fn host_snapshot_is_available_only_to_shadow_selection() -> TestResult {
|
|||||||
let mut builder = ExtensionRegistryBuilder::<()>::new();
|
let mut builder = ExtensionRegistryBuilder::<()>::new();
|
||||||
install_with_providers_and_metrics(
|
install_with_providers_and_metrics(
|
||||||
&mut builder,
|
&mut builder,
|
||||||
SkillProviders::new(),
|
SkillProviders::new().with_executor_provider(Arc::new(ExecutorProvider)),
|
||||||
Some(metrics.clone()),
|
Some(metrics.clone()),
|
||||||
|_| SkillsExtensionConfig {
|
|_| SkillsExtensionConfig {
|
||||||
include_instructions: true,
|
include_instructions: true,
|
||||||
@@ -230,6 +268,24 @@ async fn host_snapshot_is_available_only_to_shadow_selection() -> TestResult {
|
|||||||
});
|
});
|
||||||
let turn_store = ExtensionData::new("turn-host");
|
let turn_store = ExtensionData::new("turn-host");
|
||||||
turn_store.insert(HostSkillsSnapshot::new(Arc::new(outcome)));
|
turn_store.insert(HostSkillsSnapshot::new(Arc::new(outcome)));
|
||||||
|
let selected_roots = [SelectedCapabilityRoot {
|
||||||
|
id: "executor-root".to_string(),
|
||||||
|
location: CapabilityRootLocation::Environment {
|
||||||
|
environment_id: "executor".to_string(),
|
||||||
|
path: PathUri::parse("file:///skills").expect("executor skill root URI"),
|
||||||
|
},
|
||||||
|
}];
|
||||||
|
registry.context_contributors()[0]
|
||||||
|
.contribute_world_state(WorldStateContributionInput {
|
||||||
|
thread_id: codex_protocol::ThreadId::new(),
|
||||||
|
turn_id: "turn-host",
|
||||||
|
environments: &[],
|
||||||
|
ready_selected_capability_roots: &selected_roots,
|
||||||
|
session_store: &session_store,
|
||||||
|
thread_store: &thread_store,
|
||||||
|
turn_store: &turn_store,
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
|
||||||
let fragments = registry.turn_input_contributors()[0]
|
let fragments = registry.turn_input_contributors()[0]
|
||||||
.contribute(
|
.contribute(
|
||||||
|
|||||||
Reference in new issue
Block a user