Add shadow metrics for lexical skill selection (#32761)

## What changed

- Add an opt-in `skill_search` feature that ranks prompt-visible skills against each turn's user input with a bounded weighted lexical selector.
- Keep the ranked selection out of model-visible context and record metrics for selection cost, catalog reduction, and whether later implicit or `skills.read` invocations matched the ranked candidates.
- Include host-provided skills in the experiment catalog without changing the rendered skill catalog.

## Testing

- Add selector unit tests covering ranking, limits, truncation, stop words, and deterministic tie-breaking.
- Add extension tests covering turn-local invocation recording and host-skill shadow selection.

GitOrigin-RevId: 4d00a1c805ea8b391d6c6ac6a8450afa88ca3e25

codex-rs/Cargo.lock

@@ -3979,11 +3979,13 @@ dependencies = [
  "codex-exec-server",
  "codex-extension-api",
  "codex-mcp",
+ "codex-otel",
  "codex-protocol",
  "codex-tools",
  "codex-utils-absolute-path",
  "codex-utils-path-uri",
  "codex-utils-string",
+ "opentelemetry_sdk",
  "pretty_assertions",
  "schemars 0.8.22",
  "serde",

codex-rs/app-server/src/extensions.rs

@@ -84,13 +84,17 @@ where
         .with_orchestrator_provider(Arc::new(
             codex_skills_extension::OrchestratorSkillProvider::new(),
         ));
-    codex_skills_extension::install_with_providers(
+    codex_skills_extension::install_with_providers_and_metrics(
         &mut builder,
         skill_providers,
+        codex_otel::global(),
         |config: &Config| codex_skills_extension::SkillsExtensionConfig {
             include_instructions: config.include_skill_instructions,
             bundled_skills_enabled: config.bundled_skills_enabled(),
             orchestrator_skills_enabled: config.orchestrator_skills_enabled,
+            shadow_selection_enabled: config
+                .features
+                .enabled(codex_features::Feature::SkillSearch),
         },
     );
     Arc::new(builder.build())

codex-rs/core/config.schema.json

@@ -650,6 +650,9 @@
             "skill_mcp_dependency_install": {
               "type": "boolean"
             },
+            "skill_search": {
+              "type": "boolean"
+            },
             "sqlite": {
               "type": "boolean"
             },
@@ -5026,6 +5029,9 @@
         "skill_mcp_dependency_install": {
           "type": "boolean"
         },
+        "skill_search": {
+          "type": "boolean"
+        },
         "sqlite": {
           "type": "boolean"
         },

codex-rs/ext/skills/Cargo.toml

@@ -7,7 +7,6 @@ version.workspace = true
 [lib]
 name = "codex_skills_extension"
 path = "src/lib.rs"
-test = false
 doctest = false
 
 [lints]
@@ -18,6 +17,7 @@ codex-core-skills = { workspace = true }
 codex-exec-server = { workspace = true }
 codex-extension-api = { workspace = true }
 codex-mcp = { workspace = true }
+codex-otel = { workspace = true }
 codex-protocol = { workspace = true }
 codex-tools = { workspace = true }
 codex-utils-path-uri = { workspace = true }
@@ -31,5 +31,6 @@ url = { workspace = true }
 
 [dev-dependencies]
 codex-utils-absolute-path = { workspace = true }
+opentelemetry_sdk = { workspace = true }
 pretty_assertions = { workspace = true }
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }

codex-rs/ext/skills/src/config.rs

@@ -7,4 +7,6 @@ pub struct SkillsExtensionConfig {
     pub bundled_skills_enabled: bool,
     /// Whether orchestrator-owned skills are eligible for discovery.
     pub orchestrator_skills_enabled: bool,
+    /// Whether cheap skill selectors run in shadow mode without changing prompt contents.
+    pub shadow_selection_enabled: bool,
 }

codex-rs/ext/skills/src/dynamic_skill_selector.rs

@@ -0,0 +1,37 @@
+mod weighted_lexical;
+pub(crate) use weighted_lexical::WeightedLexicalSkillSelector;
+
+/// Metadata searched by a cheap skill selector.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) struct SkillSelectionDocument<'a> {
+    /// Caller-owned identifier returned in [`CheapSkillSelection::candidate_ids`].
+    pub id: usize,
+    pub name: &'a str,
+    pub short_description: Option<&'a str>,
+    pub description: &'a str,
+}
+
+/// Bounded output from one cheap skill-selection method.
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub(crate) struct CheapSkillSelection {
+    pub candidate_ids: Vec<usize>,
+    pub query_term_count: usize,
+    pub query_truncated: bool,
+    pub candidate_set_truncated: bool,
+}
+
+/// Selects likely-relevant skills without changing the model-visible catalog.
+///
+/// Implementations must be deterministic, side-effect free, and cheap enough to run in shadow
+/// mode on every turn. Callers must validate returned IDs against the supplied documents.
+pub(crate) trait CheapSkillSelector: Send + Sync {
+    /// Low-cardinality identifier suitable for experiment metrics.
+    fn method(&self) -> &'static str;
+
+    fn select(
+        &self,
+        query: &str,
+        documents: &[SkillSelectionDocument<'_>],
+        limit: usize,
+    ) -> CheapSkillSelection;
+}

codex-rs/ext/skills/src/dynamic_skill_selector/weighted_lexical.rs

@@ -0,0 +1,208 @@
+use std::collections::HashSet;
+
+use super::CheapSkillSelection;
+use super::CheapSkillSelector;
+use super::SkillSelectionDocument;
+
+const MAX_QUERY_BYTES: usize = 4 * 1024;
+const MAX_QUERY_TERMS: usize = 64;
+const MAX_DOCUMENT_BYTES: usize = 4 * 1024;
+const MAX_DOCUMENT_TERMS: usize = 256;
+const MAX_CANDIDATES: usize = 1_000;
+const MAX_RESULTS: usize = 50;
+
+const STOP_WORDS: &[&str] = &[
+    "a", "an", "and", "are", "as", "at", "be", "by", "do", "for", "from", "how", "i", "in", "is",
+    "it", "me", "my", "of", "on", "or", "please", "that", "the", "this", "to", "use", "we", "what",
+    "when", "where", "which", "with", "you", "your",
+];
+
+#[derive(Clone, Copy, Debug, Default)]
+pub(crate) struct WeightedLexicalSkillSelector;
+
+impl CheapSkillSelector for WeightedLexicalSkillSelector {
+    fn method(&self) -> &'static str {
+        "weighted_lexical_v1"
+    }
+
+    fn select(
+        &self,
+        query: &str,
+        documents: &[SkillSelectionDocument<'_>],
+        limit: usize,
+    ) -> CheapSkillSelection {
+        let (query, query_bytes_truncated) = bounded(query, MAX_QUERY_BYTES);
+        let query_phrase = normalize_phrase(query);
+        let (query_terms, query_terms_truncated) = query_terms(&query_phrase);
+        let query_truncated = query_bytes_truncated || query_terms_truncated;
+        let candidate_set_truncated = documents.len() > MAX_CANDIDATES;
+        if query_terms.is_empty() || limit == 0 {
+            return CheapSkillSelection {
+                query_term_count: query_terms.len(),
+                query_truncated,
+                candidate_set_truncated,
+                ..Default::default()
+            };
+        }
+
+        let mut scored = documents
+            .iter()
+            .take(MAX_CANDIDATES)
+            .filter_map(|document| {
+                let score = score_document(&query_phrase, &query_terms, document);
+                (score > 0).then_some((score, document.id, document.name))
+            })
+            .collect::<Vec<_>>();
+        scored.sort_by(|left, right| {
+            right
+                .0
+                .cmp(&left.0)
+                .then_with(|| left.2.cmp(right.2))
+                .then_with(|| left.1.cmp(&right.1))
+        });
+
+        CheapSkillSelection {
+            candidate_ids: scored
+                .into_iter()
+                .take(limit.min(MAX_RESULTS))
+                .map(|(_, id, _)| id)
+                .collect(),
+            query_term_count: query_terms.len(),
+            query_truncated,
+            candidate_set_truncated,
+        }
+    }
+}
+
+fn score_document(
+    query_phrase: &str,
+    query_terms: &[&str],
+    document: &SkillSelectionDocument<'_>,
+) -> u32 {
+    let name = normalize_bounded(document.name);
+    let short_description = document
+        .short_description
+        .map(normalize_bounded)
+        .unwrap_or_default();
+    let description = normalize_bounded(document.description);
+    let name_terms = phrase_terms(&name);
+    let short_description_terms = phrase_terms(&short_description);
+    let description_terms = phrase_terms(&description);
+
+    let mut score = 0u32;
+    if !name.is_empty() && contains_phrase(query_phrase, &name) {
+        score = score.saturating_add(256);
+    }
+
+    let mut matched_query_terms = 0u32;
+    for query_term in query_terms {
+        let mut matched = false;
+        if name == *query_term {
+            score = score.saturating_add(128);
+            matched = true;
+        } else if name_terms.contains(query_term) {
+            score = score.saturating_add(64);
+            matched = true;
+        } else if contains_related_term(&name_terms, query_term) {
+            score = score.saturating_add(24);
+            matched = true;
+        }
+
+        if short_description_terms.contains(query_term) {
+            score = score.saturating_add(16);
+            matched = true;
+        } else if contains_related_term(&short_description_terms, query_term) {
+            score = score.saturating_add(6);
+            matched = true;
+        }
+
+        if description_terms.contains(query_term) {
+            score = score.saturating_add(4);
+            matched = true;
+        } else if contains_related_term(&description_terms, query_term) {
+            score = score.saturating_add(1);
+            matched = true;
+        }
+
+        if matched {
+            matched_query_terms = matched_query_terms.saturating_add(1);
+        }
+    }
+
+    score.saturating_add(matched_query_terms.saturating_mul(matched_query_terms))
+}
+
+fn normalize_bounded(value: &str) -> String {
+    normalize_phrase(bounded(value, MAX_DOCUMENT_BYTES).0)
+}
+
+fn normalize_phrase(value: &str) -> String {
+    let mut normalized = String::with_capacity(value.len());
+    let mut previous_was_separator = true;
+    for character in value.chars() {
+        if character.is_alphanumeric() {
+            normalized.extend(character.to_lowercase());
+            previous_was_separator = false;
+        } else if !previous_was_separator {
+            normalized.push(' ');
+            previous_was_separator = true;
+        }
+    }
+    if previous_was_separator {
+        normalized.pop();
+    }
+    normalized
+}
+
+fn query_terms(query_phrase: &str) -> (Vec<&str>, bool) {
+    let mut seen = HashSet::new();
+    let mut terms = Vec::new();
+    for term in query_phrase
+        .split_whitespace()
+        .filter(|term| term.chars().count() >= 2 && !STOP_WORDS.contains(term))
+    {
+        if !seen.insert(term) {
+            continue;
+        }
+        if terms.len() == MAX_QUERY_TERMS {
+            return (terms, true);
+        }
+        terms.push(term);
+    }
+    (terms, false)
+}
+
+fn phrase_terms(phrase: &str) -> HashSet<&str> {
+    phrase.split_whitespace().take(MAX_DOCUMENT_TERMS).collect()
+}
+
+fn contains_phrase(haystack: &str, needle: &str) -> bool {
+    haystack == needle
+        || haystack.starts_with(&format!("{needle} "))
+        || haystack.ends_with(&format!(" {needle}"))
+        || haystack.contains(&format!(" {needle} "))
+}
+
+fn contains_related_term(terms: &HashSet<&str>, query_term: &str) -> bool {
+    if query_term.chars().count() < 4 {
+        return false;
+    }
+    terms.iter().any(|term| {
+        term.chars().count() >= 4 && (term.starts_with(query_term) || query_term.starts_with(*term))
+    })
+}
+
+fn bounded(value: &str, max_bytes: usize) -> (&str, bool) {
+    if value.len() <= max_bytes {
+        return (value, false);
+    }
+    let mut end = max_bytes;
+    while !value.is_char_boundary(end) {
+        end = end.saturating_sub(1);
+    }
+    (&value[..end], true)
+}
+
+#[cfg(test)]
+#[path = "weighted_lexical_tests.rs"]
+mod tests;

codex-rs/ext/skills/src/dynamic_skill_selector/weighted_lexical_tests.rs

@@ -0,0 +1,166 @@
+use super::*;
+use pretty_assertions::assert_eq;
+
+#[test]
+fn lexical_selector_prioritizes_an_exact_skill_name() {
+    let documents = [
+        SkillSelectionDocument {
+            id: 10,
+            name: "slides-helper",
+            short_description: None,
+            description: "Create presentations and visual decks.",
+        },
+        SkillSelectionDocument {
+            id: 20,
+            name: "presentations",
+            short_description: None,
+            description: "Create or edit PowerPoint presentations.",
+        },
+        SkillSelectionDocument {
+            id: 30,
+            name: "spreadsheets",
+            short_description: None,
+            description: "Analyze tabular data.",
+        },
+    ];
+
+    let selection = WeightedLexicalSkillSelector.select(
+        "Use presentations to create a deck",
+        &documents,
+        /*limit*/ 20,
+    );
+
+    assert_eq!(vec![20, 10], selection.candidate_ids);
+    assert!(!selection.query_truncated);
+    assert!(!selection.candidate_set_truncated);
+}
+
+#[test]
+fn lexical_selector_uses_descriptions_and_drops_zero_score_candidates() {
+    let documents = [
+        SkillSelectionDocument {
+            id: 1,
+            name: "ci-helper",
+            short_description: Some("Diagnose continuous integration failures."),
+            description: "Inspect failing GitHub Actions checks and logs.",
+        },
+        SkillSelectionDocument {
+            id: 2,
+            name: "document-editor",
+            short_description: None,
+            description: "Edit Word documents.",
+        },
+    ];
+
+    let selection = WeightedLexicalSkillSelector.select(
+        "Please diagnose the failing GitHub Actions check",
+        &documents,
+        /*limit*/ 20,
+    );
+
+    assert_eq!(vec![1], selection.candidate_ids);
+}
+
+#[test]
+fn lexical_selector_respects_requested_limit() {
+    let names = (0..10)
+        .map(|index| format!("lint-{index}"))
+        .collect::<Vec<_>>();
+    let documents = names
+        .iter()
+        .enumerate()
+        .map(|(id, name)| SkillSelectionDocument {
+            id,
+            name,
+            short_description: None,
+            description: "Fix lint errors.",
+        })
+        .collect::<Vec<_>>();
+
+    let selection =
+        WeightedLexicalSkillSelector.select("fix lint errors", &documents, /*limit*/ 3);
+
+    assert_eq!(3, selection.candidate_ids.len());
+}
+
+#[test]
+fn lexical_selector_reports_bounded_inputs() {
+    let long_query = "match ".repeat(MAX_QUERY_BYTES);
+    let names = (0..=MAX_CANDIDATES)
+        .map(|index| format!("candidate-{index}"))
+        .collect::<Vec<_>>();
+    let documents = names
+        .iter()
+        .enumerate()
+        .map(|(id, name)| SkillSelectionDocument {
+            id,
+            name,
+            short_description: None,
+            description: "match",
+        })
+        .collect::<Vec<_>>();
+
+    let selection = WeightedLexicalSkillSelector.select(&long_query, &documents, /*limit*/ 20);
+
+    assert!(selection.query_truncated);
+    assert!(selection.candidate_set_truncated);
+    assert_eq!(20, selection.candidate_ids.len());
+    assert!(
+        selection
+            .candidate_ids
+            .iter()
+            .all(|id| *id < MAX_CANDIDATES)
+    );
+}
+
+#[test]
+fn lexical_selector_caps_query_terms() {
+    let query = (0..=MAX_QUERY_TERMS)
+        .map(|index| format!("term{index}"))
+        .collect::<Vec<_>>()
+        .join(" ");
+    let documents = [SkillSelectionDocument {
+        id: 1,
+        name: "term0",
+        short_description: None,
+        description: "term0",
+    }];
+
+    let selection = WeightedLexicalSkillSelector.select(&query, &documents, /*limit*/ 20);
+
+    assert_eq!(MAX_QUERY_TERMS, selection.query_term_count);
+    assert!(selection.query_truncated);
+}
+
+#[test]
+fn lexical_selector_returns_nothing_for_stop_words_only() {
+    let documents = [SkillSelectionDocument {
+        id: 1,
+        name: "anything",
+        short_description: None,
+        description: "Do anything.",
+    }];
+
+    let selection =
+        WeightedLexicalSkillSelector.select("please use the", &documents, /*limit*/ 20);
+
+    assert_eq!(CheapSkillSelection::default(), selection);
+}
+
+#[test]
+fn selector_can_be_used_behind_the_shared_trait() {
+    fn run(selector: &dyn CheapSkillSelector) -> CheapSkillSelection {
+        selector.select(
+            "review code",
+            &[SkillSelectionDocument {
+                id: 7,
+                name: "code-review",
+                short_description: None,
+                description: "Review code.",
+            }],
+            /*limit*/ 20,
+        )
+    }
+
+    assert_eq!(vec![7], run(&WeightedLexicalSkillSelector).candidate_ids);
+}

codex-rs/ext/skills/src/extension.rs

@@ -11,6 +11,9 @@ use codex_extension_api::ExtensionEventSink;
 use codex_extension_api::ExtensionFuture;
 use codex_extension_api::ExtensionRegistryBuilder;
 use codex_extension_api::PromptFragment;
+use codex_extension_api::SkillInvocationContributor;
+use codex_extension_api::SkillInvocationInput;
+use codex_extension_api::SkillInvocationKind;
 use codex_extension_api::ThreadLifecycleContributor;
 use codex_extension_api::ThreadStartInput;
 use codex_extension_api::ToolCall;
@@ -21,6 +24,7 @@ use codex_extension_api::TurnInputContributor;
 use codex_extension_api::WorldStateContributionInput;
 use codex_extension_api::WorldStateSectionContribution;
 use codex_mcp::McpResourceClient;
+use codex_otel::MetricsClient;
 use codex_protocol::openai_models::ModelInfo;
 use codex_protocol::protocol::Event;
 use codex_protocol::protocol::EventMsg;
@@ -34,6 +38,7 @@ use crate::catalog::SkillSourceKind;
 use crate::fragments::SkillInstructions;
 use crate::provider::HostSkillProvider;
 use crate::provider::SkillListQuery;
+use crate::provider::SkillProvider;
 use crate::provider::SkillReadRequest;
 use crate::render::MAX_SKILL_NAME_BYTES;
 use crate::render::MAX_SKILL_PATH_BYTES;
@@ -41,6 +46,7 @@ use crate::render::available_skills_fragment;
 use crate::render::truncate_main_prompt_contents;
 use crate::render::truncate_utf8_to_bytes;
 use crate::selection::collect_explicit_skill_mentions;
+use crate::shadow_selection_experiment::ShadowSelectionExperiment;
 use crate::sources::SkillProviders;
 use crate::state::ExecutorSkillsStepState;
 use crate::state::SkillsThreadState;
@@ -52,6 +58,7 @@ struct SkillsExtension<C> {
     providers: SkillProviders,
     event_sink: Arc<dyn ExtensionEventSink>,
     config_from_host: Arc<dyn Fn(&C) -> SkillsExtensionConfig + Send + Sync>,
+    shadow_selection: Arc<ShadowSelectionExperiment>,
 }
 
 impl<C> ThreadLifecycleContributor<C> for SkillsExtension<C>
@@ -201,10 +208,37 @@ where
             self.providers.clone(),
             session_store.get::<McpResourceClient>(),
             thread_state,
+            Arc::clone(&self.shadow_selection),
         )
     }
 }
 
+impl<C> SkillInvocationContributor for SkillsExtension<C>
+where
+    C: Send + Sync + 'static,
+{
+    fn on_skill_invocation<'a>(
+        &'a self,
+        input: SkillInvocationInput<'a>,
+    ) -> ExtensionFuture<'a, ()> {
+        Box::pin(async move {
+            match input.kind {
+                SkillInvocationKind::Implicit => {
+                    if let Some(state) = input
+                        .thread_store
+                        .get::<SkillsThreadState>()
+                        .and_then(|state| state.shadow_selection_turn(input.turn_id))
+                    {
+                        self.shadow_selection
+                            .record_invocation(&state, input.skill_resource);
+                    }
+                }
+                SkillInvocationKind::Explicit => {}
+            }
+        })
+    }
+}
+
 impl<C> TurnInputContributor for SkillsExtension<C>
 where
     C: Send + Sync + 'static,
@@ -232,6 +266,7 @@ where
                 include_orchestrator_skills: thread_state.orchestrator_skills_enabled(),
                 mcp_resources: session_store.get::<McpResourceClient>(),
             };
+            let host_query = query.clone();
             let mut catalog = self.list_skills(query, &thread_state).await;
             if let Some(executor_skills) = turn_store.get::<ExecutorSkillsStepState>() {
                 catalog.extend(executor_skills.0.clone());
@@ -241,6 +276,24 @@ where
             }
 
             let selected_entries = collect_explicit_skill_mentions(&input.user_input, &catalog);
+            let shadow_selection_turn = if config.shadow_selection_enabled {
+                // App-server leaves host discovery in core, so its configured providers omit the
+                // host snapshot. Extend only the experiment catalog to keep rendered context intact.
+                let mut shadow_catalog = catalog.clone();
+                if host_snapshot.is_some()
+                    && let Ok(host_catalog) = HostSkillProvider::new().list(host_query).await
+                {
+                    shadow_catalog.extend(host_catalog);
+                }
+                Some(
+                    self.shadow_selection
+                        .run(&input.user_input, &shadow_catalog),
+                )
+            } else {
+                None
+            };
+            thread_state
+                .replace_shadow_selection_turn(input.turn_id.clone(), shadow_selection_turn);
             let mut fragments: Vec<Box<dyn ContextualUserFragment + Send>> = Vec::new();
             if config.include_instructions {
                 let mut turn_catalog = catalog.clone();
@@ -406,15 +459,33 @@ pub fn install_with_providers<C>(
     config_from_host: impl Fn(&C) -> SkillsExtensionConfig + Send + Sync + 'static,
 ) where
     C: Send + Sync + 'static,
+{
+    install_with_providers_and_metrics(
+        registry,
+        providers,
+        /*metrics_client*/ None,
+        config_from_host,
+    );
+}
+
+pub fn install_with_providers_and_metrics<C>(
+    registry: &mut ExtensionRegistryBuilder<C>,
+    providers: SkillProviders,
+    metrics_client: Option<MetricsClient>,
+    config_from_host: impl Fn(&C) -> SkillsExtensionConfig + Send + Sync + 'static,
+) where
+    C: Send + Sync + 'static,
 {
     let extension = Arc::new(SkillsExtension {
         providers,
         event_sink: registry.event_sink(),
         config_from_host: Arc::new(config_from_host),
+        shadow_selection: Arc::new(ShadowSelectionExperiment::new(metrics_client)),
     });
     registry.thread_lifecycle_contributor(extension.clone());
     registry.config_contributor(extension.clone());
     registry.prompt_contributor(extension.clone());
     registry.turn_input_contributor(extension.clone());
+    registry.skill_invocation_contributor(extension.clone());
     registry.tool_contributor(extension);
 }

codex-rs/ext/skills/src/lib.rs

@@ -1,10 +1,12 @@
 pub mod catalog;
 mod config;
+mod dynamic_skill_selector;
 mod extension;
 mod fragments;
 pub mod provider;
 mod render;
 mod selection;
+mod shadow_selection_experiment;
 mod sources;
 mod state;
 mod tools;
@@ -13,6 +15,7 @@ mod world_state;
 pub use config::SkillsExtensionConfig;
 pub use extension::install;
 pub use extension::install_with_providers;
+pub use extension::install_with_providers_and_metrics;
 pub use provider::ExecutorSkillProvider;
 pub use provider::HostSkillProvider;
 pub use provider::OrchestratorSkillProvider;

codex-rs/ext/skills/src/shadow_selection_experiment.rs

@@ -0,0 +1,356 @@
+// This shadow-selection experiment is temporary and should be removed after evaluation.
+
+use std::collections::HashSet;
+use std::sync::Mutex;
+use std::sync::PoisonError;
+use std::time::Duration;
+use std::time::Instant;
+
+use codex_otel::MetricsClient;
+use codex_protocol::user_input::UserInput;
+
+use crate::catalog::SkillCatalog;
+use crate::dynamic_skill_selector::CheapSkillSelection;
+use crate::dynamic_skill_selector::CheapSkillSelector;
+use crate::dynamic_skill_selector::SkillSelectionDocument;
+use crate::dynamic_skill_selector::WeightedLexicalSkillSelector;
+
+const MAX_SHADOW_QUERY_BYTES: usize = 16 * 1024;
+const MAX_SHADOW_RESULTS: usize = 20;
+
+const RUN_METRIC: &str = "codex.skills.shadow_selection";
+const DURATION_METRIC: &str = "codex.skills.shadow_selection.duration_ms";
+const CATALOG_ENTRY_COUNT_METRIC: &str = "codex.skills.shadow_selection.catalog_entries";
+const SELECTED_ENTRY_COUNT_METRIC: &str = "codex.skills.shadow_selection.selected_entries";
+const QUERY_TERM_COUNT_METRIC: &str = "codex.skills.shadow_selection.query_terms";
+const REDUCTION_BPS_METRIC: &str = "codex.skills.shadow_selection.reduction_bps";
+const INVOCATION_METRIC: &str = "codex.skills.shadow_selection.invocation";
+
+pub(crate) struct ShadowSelectionExperiment {
+    selectors: Vec<Box<dyn CheapSkillSelector>>,
+    metrics_client: Option<MetricsClient>,
+}
+
+impl ShadowSelectionExperiment {
+    pub(crate) fn new(metrics_client: Option<MetricsClient>) -> Self {
+        Self {
+            selectors: vec![Box::new(WeightedLexicalSkillSelector)],
+            metrics_client,
+        }
+    }
+
+    pub(crate) fn run(
+        &self,
+        inputs: &[UserInput],
+        catalog: &SkillCatalog,
+    ) -> ShadowSelectionTurnState {
+        let query = build_shadow_query(inputs);
+        let query_script = query_script_tag(&query.text);
+        let documents = catalog
+            .entries
+            .iter()
+            .enumerate()
+            .filter(|(_, entry)| entry.enabled && entry.prompt_visible)
+            .map(|(id, entry)| SkillSelectionDocument {
+                id,
+                name: entry.name.as_str(),
+                short_description: entry.short_description.as_deref(),
+                description: entry.description.as_str(),
+            })
+            .collect::<Vec<_>>();
+        let eligible_ids = documents
+            .iter()
+            .map(|document| document.id)
+            .collect::<HashSet<_>>();
+        let mut ranked_selections = Vec::with_capacity(self.selectors.len());
+
+        for selector in &self.selectors {
+            let start = Instant::now();
+            let selection =
+                selector.select(&query.text, &documents, /*limit*/ MAX_SHADOW_RESULTS);
+            let duration = start.elapsed();
+            let selected_ids = sanitize_selected_ids(&selection, &eligible_ids);
+            self.record_metrics(ShadowSelectionObservation {
+                method: selector.method(),
+                selection: &selection,
+                query_truncated_before_selection: query.truncated,
+                query_script,
+                catalog_entry_count: documents.len(),
+                selected_entry_count: selected_ids.len(),
+                duration,
+            });
+            ranked_selections.push(RankedSelection {
+                method: selector.method(),
+                skill_resources: selected_ids
+                    .iter()
+                    .map(|id| normalize_skill_resource(catalog.entries[*id].main_prompt.as_str()))
+                    .collect(),
+            });
+            tracing::debug!(
+                method = selector.method(),
+                catalog_entries = documents.len(),
+                selected_entries = selected_ids.len(),
+                query_terms = selection.query_term_count,
+                query_script,
+                query_truncated = query.truncated || selection.query_truncated,
+                candidate_set_truncated = selection.candidate_set_truncated,
+                "ran shadow skill selection"
+            );
+        }
+
+        ShadowSelectionTurnState {
+            ranked_selections,
+            query_script,
+            seen_skill_resources: Mutex::new(HashSet::new()),
+        }
+    }
+
+    pub(crate) fn record_invocation(&self, state: &ShadowSelectionTurnState, skill_resource: &str) {
+        let skill_resource = normalize_skill_resource(skill_resource);
+        if !state
+            .seen_skill_resources
+            .lock()
+            .unwrap_or_else(PoisonError::into_inner)
+            .insert(skill_resource.clone())
+        {
+            return;
+        }
+        let Some(metrics_client) = self.metrics_client.as_ref() else {
+            return;
+        };
+        for selection in &state.ranked_selections {
+            let rank = selection
+                .skill_resources
+                .iter()
+                .position(|candidate| candidate == &skill_resource)
+                .map(|index| index + 1);
+            let tags = [
+                ("method", selection.method),
+                ("hit", bool_tag(rank.is_some())),
+                ("rank", rank_bucket(rank)),
+                ("query_script", state.query_script),
+            ];
+            let _ = metrics_client.counter(INVOCATION_METRIC, /*inc*/ 1, &tags);
+        }
+    }
+
+    fn record_metrics(&self, observation: ShadowSelectionObservation<'_>) {
+        let Some(metrics_client) = self.metrics_client.as_ref() else {
+            return;
+        };
+        let ShadowSelectionObservation {
+            method,
+            selection,
+            query_truncated_before_selection,
+            query_script,
+            catalog_entry_count,
+            selected_entry_count,
+            duration,
+        } = observation;
+        let status = selection_status(selection, selected_entry_count);
+        let query_truncated =
+            bool_tag(query_truncated_before_selection || selection.query_truncated);
+        let candidate_set_truncated = bool_tag(selection.candidate_set_truncated);
+        let tags = [
+            ("method", method),
+            ("status", status),
+            ("query_script", query_script),
+            ("query_truncated", query_truncated),
+            ("candidate_set_truncated", candidate_set_truncated),
+        ];
+        let _ = metrics_client.counter(RUN_METRIC, /*inc*/ 1, &tags);
+        let _ = metrics_client.record_duration(DURATION_METRIC, duration, &tags);
+        let _ = metrics_client.histogram(
+            CATALOG_ENTRY_COUNT_METRIC,
+            metric_value(catalog_entry_count),
+            &tags,
+        );
+        let _ = metrics_client.histogram(
+            SELECTED_ENTRY_COUNT_METRIC,
+            metric_value(selected_entry_count),
+            &tags,
+        );
+        let _ = metrics_client.histogram(
+            QUERY_TERM_COUNT_METRIC,
+            metric_value(selection.query_term_count),
+            &tags,
+        );
+        let _ = metrics_client.histogram(
+            REDUCTION_BPS_METRIC,
+            reduction_bps(catalog_entry_count, selected_entry_count),
+            &tags,
+        );
+    }
+}
+
+pub(crate) struct ShadowSelectionTurnState {
+    ranked_selections: Vec<RankedSelection>,
+    query_script: &'static str,
+    seen_skill_resources: Mutex<HashSet<String>>,
+}
+
+struct RankedSelection {
+    method: &'static str,
+    skill_resources: Vec<String>,
+}
+
+struct ShadowSelectionObservation<'a> {
+    method: &'static str,
+    selection: &'a CheapSkillSelection,
+    query_truncated_before_selection: bool,
+    query_script: &'static str,
+    catalog_entry_count: usize,
+    selected_entry_count: usize,
+    duration: Duration,
+}
+
+fn sanitize_selected_ids(
+    selection: &CheapSkillSelection,
+    eligible_ids: &HashSet<usize>,
+) -> Vec<usize> {
+    let mut seen = HashSet::new();
+    selection
+        .candidate_ids
+        .iter()
+        .copied()
+        .filter(|id| eligible_ids.contains(id) && seen.insert(*id))
+        .take(MAX_SHADOW_RESULTS)
+        .collect()
+}
+
+fn selection_status(selection: &CheapSkillSelection, selected_entry_count: usize) -> &'static str {
+    if selection.query_term_count == 0 {
+        "no_query_terms"
+    } else if selected_entry_count == 0 {
+        "no_matches"
+    } else {
+        "selected"
+    }
+}
+
+fn reduction_bps(catalog_entry_count: usize, selected_entry_count: usize) -> i64 {
+    if catalog_entry_count == 0 {
+        return 0;
+    }
+    10_000i64.saturating_sub(ratio_bps(selected_entry_count, catalog_entry_count))
+}
+
+fn ratio_bps(numerator: usize, denominator: usize) -> i64 {
+    if denominator == 0 {
+        return 0;
+    }
+    let numerator = u128::try_from(numerator).unwrap_or(u128::MAX);
+    let denominator = u128::try_from(denominator).unwrap_or(u128::MAX);
+    let basis_points = numerator.saturating_mul(10_000) / denominator;
+    i64::try_from(basis_points).unwrap_or(i64::MAX)
+}
+
+fn metric_value(value: usize) -> i64 {
+    i64::try_from(value).unwrap_or(i64::MAX)
+}
+
+fn bool_tag(value: bool) -> &'static str {
+    if value { "true" } else { "false" }
+}
+
+fn rank_bucket(rank: Option<usize>) -> &'static str {
+    match rank {
+        Some(1) => "1",
+        Some(2..=5) => "2_5",
+        Some(6..=10) => "6_10",
+        Some(11..=MAX_SHADOW_RESULTS) => "11_20",
+        Some(_) | None => "miss",
+    }
+}
+
+fn normalize_skill_resource(skill_resource: &str) -> String {
+    skill_resource.replace('\\', "/")
+}
+
+fn query_script_tag(query: &str) -> &'static str {
+    let mut has_ascii_latin = false;
+    let mut has_cjk = false;
+    let mut has_other = false;
+
+    for character in query.chars().filter(|character| character.is_alphabetic()) {
+        if character.is_ascii_alphabetic() {
+            has_ascii_latin = true;
+        } else if is_cjk(character) {
+            has_cjk = true;
+        } else {
+            has_other = true;
+        }
+    }
+
+    match (has_ascii_latin, has_cjk, has_other) {
+        (false, false, false) => "none",
+        (true, false, false) => "ascii_latin",
+        (false, true, false) => "cjk",
+        (false, false, true) => "other",
+        (true, true, false) | (true, false, true) | (false, true, true) | (true, true, true) => {
+            "mixed"
+        }
+    }
+}
+
+fn is_cjk(character: char) -> bool {
+    matches!(
+        character,
+        '\u{1100}'..='\u{11ff}'
+            | '\u{3040}'..='\u{30ff}'
+            | '\u{3100}'..='\u{312f}'
+            | '\u{3130}'..='\u{318f}'
+            | '\u{31a0}'..='\u{31bf}'
+            | '\u{31f0}'..='\u{31ff}'
+            | '\u{3400}'..='\u{4dbf}'
+            | '\u{4e00}'..='\u{9fff}'
+            | '\u{a960}'..='\u{a97f}'
+            | '\u{ac00}'..='\u{d7af}'
+            | '\u{d7b0}'..='\u{d7ff}'
+            | '\u{f900}'..='\u{faff}'
+            | '\u{20000}'..='\u{2fa1f}'
+    )
+}
+
+struct ShadowQuery {
+    text: String,
+    truncated: bool,
+}
+
+fn build_shadow_query(inputs: &[UserInput]) -> ShadowQuery {
+    let mut text = String::new();
+    let mut truncated = false;
+    for input in inputs {
+        let part = match input {
+            UserInput::Text { text, .. } => text.as_str(),
+            UserInput::Skill { name, .. } | UserInput::Mention { name, .. } => name.as_str(),
+            _ => continue,
+        };
+        if part.is_empty() {
+            continue;
+        }
+        if !text.is_empty() && !push_bounded(&mut text, " ") {
+            truncated = true;
+            break;
+        }
+        if !push_bounded(&mut text, part) {
+            truncated = true;
+            break;
+        }
+    }
+    ShadowQuery { text, truncated }
+}
+
+fn push_bounded(destination: &mut String, value: &str) -> bool {
+    let remaining = MAX_SHADOW_QUERY_BYTES.saturating_sub(destination.len());
+    if value.len() <= remaining {
+        destination.push_str(value);
+        return true;
+    }
+    let mut end = remaining;
+    while !value.is_char_boundary(end) {
+        end = end.saturating_sub(1);
+    }
+    destination.push_str(&value[..end]);
+    false
+}

codex-rs/ext/skills/src/state.rs

@@ -20,6 +20,7 @@ use crate::catalog::SkillResourceId;
 use crate::catalog::SkillSourceKind;
 use crate::provider::SkillListQuery;
 use crate::provider::SkillReadRequest;
+use crate::shadow_selection_experiment::ShadowSelectionTurnState;
 use crate::sources::SkillProviders;
 
 const MAX_CACHED_ORCHESTRATOR_RESOURCES: usize = 100;
@@ -30,6 +31,7 @@ pub(crate) struct SkillsThreadState {
     orchestrator_skills_available: bool,
     executor_cache: Mutex<Vec<CachedExecutorCatalog>>,
     orchestrator_cache: Mutex<Option<Arc<OrchestratorGenerationCache>>>,
+    shadow_selection_turn: Mutex<Option<ShadowSelectionTurn>>,
 }
 
 impl SkillsThreadState {
@@ -39,6 +41,7 @@ impl SkillsThreadState {
             orchestrator_skills_available,
             executor_cache: Mutex::new(Vec::new()),
             orchestrator_cache: Mutex::new(None),
+            shadow_selection_turn: Mutex::new(None),
         }
     }
 
@@ -60,6 +63,33 @@ impl SkillsThreadState {
         self.orchestrator_skills_available && self.config().orchestrator_skills_enabled
     }
 
+    pub(crate) fn replace_shadow_selection_turn(
+        &self,
+        turn_id: String,
+        state: Option<ShadowSelectionTurnState>,
+    ) {
+        *self
+            .shadow_selection_turn
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner) =
+            state.map(|state| ShadowSelectionTurn {
+                turn_id,
+                state: Arc::new(state),
+            });
+    }
+
+    pub(crate) fn shadow_selection_turn(
+        &self,
+        turn_id: &str,
+    ) -> Option<Arc<ShadowSelectionTurnState>> {
+        self.shadow_selection_turn
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .as_ref()
+            .filter(|turn| turn.turn_id == turn_id)
+            .map(|turn| Arc::clone(&turn.state))
+    }
+
     /// Returns catalogs for stable selected roots.
     ///
     /// The first catalog returned for a root remains cached until this thread state is dropped.
@@ -196,6 +226,11 @@ impl SkillsThreadState {
     }
 }
 
+struct ShadowSelectionTurn {
+    turn_id: String,
+    state: Arc<ShadowSelectionTurnState>,
+}
+
 struct CachedExecutorCatalog {
     root: SelectedCapabilityRoot,
     catalog: SkillCatalog,

codex-rs/ext/skills/src/tools/mod.rs

@@ -23,6 +23,7 @@ use crate::catalog::SkillAuthority;
 use crate::catalog::SkillCatalog;
 use crate::catalog::SkillSourceKind;
 use crate::provider::SkillListQuery;
+use crate::shadow_selection_experiment::ShadowSelectionExperiment;
 use crate::sources::SkillProviders;
 use crate::state::SkillsThreadState;
 
@@ -37,11 +38,13 @@ pub(crate) fn skill_tools(
     providers: SkillProviders,
     mcp_resources: Option<Arc<McpResourceClient>>,
     thread_state: Arc<SkillsThreadState>,
+    shadow_selection: Arc<ShadowSelectionExperiment>,
 ) -> Vec<Arc<dyn ToolExecutor<ToolCall>>> {
     let context = SkillToolContext {
         providers,
         mcp_resources,
         thread_state,
+        shadow_selection,
     };
     vec![
         Arc::new(list::ListTool {
@@ -56,6 +59,7 @@ struct SkillToolContext {
     providers: SkillProviders,
     mcp_resources: Option<Arc<McpResourceClient>>,
     thread_state: Arc<SkillsThreadState>,
+    shadow_selection: Arc<ShadowSelectionExperiment>,
 }
 
 impl SkillToolContext {

codex-rs/ext/skills/src/tools/read.rs

@@ -63,14 +63,14 @@ impl ToolExecutor<ToolCall> for ReadTool {
             validate_handle("resource", &args.resource, MAX_HANDLE_BYTES)?;
 
             let catalog = self.context.catalog(&call.turn_id, args.authority).await;
-            let package_is_available = catalog.entries.iter().any(|entry| {
+            let Some(skill_entry) = catalog.entries.iter().find(|entry| {
                 entry.enabled && entry.authority == authority && entry.id.0 == args.package
-            });
+            }) else {
-            if !package_is_available {
                 return Err(FunctionCallError::RespondToModel(
                     "skill package is not available from the requested authority".to_string(),
                 ));
-            }
+            };
+            let main_prompt = skill_entry.main_prompt.clone();
 
             let requested_resource = SkillResourceId::new(args.resource);
             let result = self
@@ -103,6 +103,16 @@ impl ToolExecutor<ToolCall> for ReadTool {
                 ));
             }
 
+            if let Some(state) = self
+                .context
+                .thread_state
+                .shadow_selection_turn(&call.turn_id)
+            {
+                self.context
+                    .shadow_selection
+                    .record_invocation(&state, main_prompt.as_str());
+            }
+
             external_json_output(&ReadResponse {
                 resource: result.resource.as_str().to_string(),
                 contents: result.contents,

codex-rs/ext/skills/tests/implicit_invocation.rs

@@ -0,0 +1,331 @@
+use std::collections::BTreeMap;
+use std::sync::Arc;
+
+use codex_core_skills::HostSkillsSnapshot;
+use codex_core_skills::SkillLoadOutcome;
+use codex_core_skills::SkillMetadata;
+use codex_extension_api::ConversationHistory;
+use codex_extension_api::ExtensionData;
+use codex_extension_api::ExtensionRegistry;
+use codex_extension_api::ExtensionRegistryBuilder;
+use codex_extension_api::NoopTurnItemEmitter;
+use codex_extension_api::SkillInvocationInput;
+use codex_extension_api::SkillInvocationKind;
+use codex_extension_api::ThreadStartInput;
+use codex_extension_api::ToolCall;
+use codex_extension_api::ToolPayload;
+use codex_extension_api::TurnInputContext;
+use codex_mcp::CODEX_APPS_MCP_SERVER_NAME;
+use codex_otel::MetricsClient;
+use codex_otel::MetricsConfig;
+use codex_protocol::protocol::SessionSource;
+use codex_protocol::protocol::SkillScope;
+use codex_protocol::protocol::TruncationPolicy;
+use codex_protocol::user_input::UserInput;
+use codex_skills_extension::SkillProviders;
+use codex_skills_extension::SkillsExtensionConfig;
+use codex_skills_extension::catalog::SkillAuthority;
+use codex_skills_extension::catalog::SkillCatalog;
+use codex_skills_extension::catalog::SkillCatalogEntry;
+use codex_skills_extension::catalog::SkillPackageId;
+use codex_skills_extension::catalog::SkillReadResult;
+use codex_skills_extension::catalog::SkillResourceId;
+use codex_skills_extension::catalog::SkillSearchResult;
+use codex_skills_extension::catalog::SkillSourceKind;
+use codex_skills_extension::install_with_providers_and_metrics;
+use codex_skills_extension::provider::SkillListQuery;
+use codex_skills_extension::provider::SkillProvider;
+use codex_skills_extension::provider::SkillProviderFuture;
+use codex_skills_extension::provider::SkillReadRequest;
+use codex_skills_extension::provider::SkillSearchRequest;
+use codex_utils_absolute_path::AbsolutePathBuf;
+use opentelemetry_sdk::metrics::InMemoryMetricExporter;
+use opentelemetry_sdk::metrics::data::AggregatedMetrics;
+use opentelemetry_sdk::metrics::data::MetricData;
+use pretty_assertions::assert_eq;
+
+type TestResult = Result<(), Box<dyn std::error::Error>>;
+type InvocationPoint = (BTreeMap<String, String>, u64);
+
+const INVOCATION_METRIC: &str = "codex.skills.shadow_selection.invocation";
+const PACKAGE: &str = "orchestrator/demo";
+const MAIN_RESOURCE: &str = "skill://orchestrator/demo/SKILL.md";
+
+#[derive(Clone)]
+struct OrchestratorProvider;
+
+impl SkillProvider for OrchestratorProvider {
+    fn list(&self, _query: SkillListQuery) -> SkillProviderFuture<'_, SkillCatalog> {
+        Box::pin(std::future::ready(Ok(SkillCatalog {
+            entries: vec![SkillCatalogEntry::new(
+                SkillPackageId(PACKAGE.to_string()),
+                SkillAuthority::new(SkillSourceKind::Orchestrator, CODEX_APPS_MCP_SERVER_NAME),
+                "demo",
+                "Use the demo skill.",
+                SkillResourceId::new(MAIN_RESOURCE),
+            )],
+            warnings: Vec::new(),
+        })))
+    }
+
+    fn read(&self, request: SkillReadRequest) -> SkillProviderFuture<'_, SkillReadResult> {
+        Box::pin(std::future::ready(Ok(SkillReadResult {
+            resource: request.resource,
+            contents: "# Demo\n\nUse the demo skill.".to_string(),
+        })))
+    }
+
+    fn search(&self, _request: SkillSearchRequest) -> SkillProviderFuture<'_, SkillSearchResult> {
+        Box::pin(std::future::ready(Ok(SkillSearchResult::default())))
+    }
+}
+
+#[tokio::test]
+async fn implicit_core_and_native_read_invocations_share_turn_local_recording() -> TestResult {
+    let metrics = MetricsClient::new(
+        MetricsConfig::in_memory(
+            "test",
+            "codex-skills-extension",
+            env!("CARGO_PKG_VERSION"),
+            InMemoryMetricExporter::default(),
+        )
+        .with_runtime_reader(),
+    )?;
+    let mut builder = ExtensionRegistryBuilder::<()>::new();
+    install_with_providers_and_metrics(
+        &mut builder,
+        SkillProviders::new().with_orchestrator_provider(Arc::new(OrchestratorProvider)),
+        Some(metrics.clone()),
+        |_| SkillsExtensionConfig {
+            include_instructions: false,
+            bundled_skills_enabled: false,
+            orchestrator_skills_enabled: true,
+            shadow_selection_enabled: true,
+        },
+    );
+    let registry = builder.build();
+    let session_store = ExtensionData::new("session");
+    let thread_store = ExtensionData::new("thread");
+    registry.thread_lifecycle_contributors()[0]
+        .on_thread_start(ThreadStartInput {
+            config: &(),
+            session_source: &SessionSource::Cli,
+            persistent_thread_state_available: true,
+            environments: &[],
+            session_store: &session_store,
+            thread_store: &thread_store,
+        })
+        .await;
+
+    let core_turn = start_turn(&registry, &session_store, &thread_store, "turn-core").await;
+    registry.skill_invocation_contributors()[0]
+        .on_skill_invocation(SkillInvocationInput {
+            session_store: &session_store,
+            thread_store: &thread_store,
+            turn_store: core_turn.as_ref(),
+            turn_id: "turn-core",
+            skill_resource: MAIN_RESOURCE,
+            kind: SkillInvocationKind::Implicit,
+        })
+        .await;
+
+    start_turn(&registry, &session_store, &thread_store, "turn-tool").await;
+    let tools = registry.tool_contributors()[0].tools(&session_store, &thread_store);
+    let read_tool = tools
+        .iter()
+        .find(|tool| tool.tool_name().name == "read")
+        .ok_or("skills.read tool should be registered")?;
+    for call_id in ["call-1", "call-2"] {
+        read_tool
+            .handle(ToolCall {
+                turn_id: "turn-tool".to_string(),
+                call_id: call_id.to_string(),
+                tool_name: read_tool.tool_name(),
+                model: "gpt-test".to_string(),
+                truncation_policy: TruncationPolicy::Bytes(1_024),
+                conversation_history: ConversationHistory::default(),
+                turn_item_emitter: Arc::new(NoopTurnItemEmitter),
+                environments: Vec::new(),
+                payload: ToolPayload::Function {
+                    arguments: serde_json::json!({
+                        "authority": {"kind": "orchestrator"},
+                        "package": PACKAGE,
+                        "resource": MAIN_RESOURCE,
+                    })
+                    .to_string(),
+                },
+            })
+            .await?;
+    }
+
+    assert_eq!(
+        vec![(
+            BTreeMap::from([
+                ("hit".to_string(), "true".to_string()),
+                ("method".to_string(), "weighted_lexical_v1".to_string()),
+                ("query_script".to_string(), "ascii_latin".to_string()),
+                ("rank".to_string(), "1".to_string()),
+            ]),
+            2,
+        )],
+        invocation_points(&metrics)?,
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn host_snapshot_is_available_only_to_shadow_selection() -> TestResult {
+    let metrics = MetricsClient::new(
+        MetricsConfig::in_memory(
+            "test",
+            "codex-skills-extension",
+            env!("CARGO_PKG_VERSION"),
+            InMemoryMetricExporter::default(),
+        )
+        .with_runtime_reader(),
+    )?;
+    let mut builder = ExtensionRegistryBuilder::<()>::new();
+    install_with_providers_and_metrics(
+        &mut builder,
+        SkillProviders::new(),
+        Some(metrics.clone()),
+        |_| SkillsExtensionConfig {
+            include_instructions: true,
+            bundled_skills_enabled: false,
+            orchestrator_skills_enabled: false,
+            shadow_selection_enabled: true,
+        },
+    );
+    let registry = builder.build();
+    let session_store = ExtensionData::new("session");
+    let thread_store = ExtensionData::new("thread");
+    registry.thread_lifecycle_contributors()[0]
+        .on_thread_start(ThreadStartInput {
+            config: &(),
+            session_source: &SessionSource::Cli,
+            persistent_thread_state_available: true,
+            environments: &[],
+            session_store: &session_store,
+            thread_store: &thread_store,
+        })
+        .await;
+
+    let skill_path = AbsolutePathBuf::try_from(
+        std::env::temp_dir()
+            .join("codex-shadow-host-skill")
+            .join("SKILL.md"),
+    )?;
+    let skill_resource = skill_path.to_string_lossy().into_owned();
+    let mut outcome = SkillLoadOutcome::default();
+    outcome.skills.push(SkillMetadata {
+        name: "演示文稿".to_string(),
+        description: "创建演示文稿。".to_string(),
+        short_description: None,
+        interface: None,
+        dependencies: None,
+        policy: None,
+        path_to_skills_md: skill_path,
+        scope: SkillScope::User,
+        plugin_id: None,
+    });
+    let turn_store = ExtensionData::new("turn-host");
+    turn_store.insert(HostSkillsSnapshot::new(Arc::new(outcome)));
+
+    let fragments = registry.turn_input_contributors()[0]
+        .contribute(
+            TurnInputContext {
+                turn_id: "turn-host".to_string(),
+                user_input: vec![UserInput::Text {
+                    text: "演示文稿".to_string(),
+                    text_elements: Vec::new(),
+                }],
+                environments: Vec::new(),
+            },
+            &session_store,
+            &thread_store,
+            &turn_store,
+        )
+        .await;
+    assert!(fragments.is_empty());
+
+    registry.skill_invocation_contributors()[0]
+        .on_skill_invocation(SkillInvocationInput {
+            session_store: &session_store,
+            thread_store: &thread_store,
+            turn_store: &turn_store,
+            turn_id: "turn-host",
+            skill_resource: &skill_resource,
+            kind: SkillInvocationKind::Implicit,
+        })
+        .await;
+
+    assert_eq!(
+        vec![(
+            BTreeMap::from([
+                ("hit".to_string(), "true".to_string()),
+                ("method".to_string(), "weighted_lexical_v1".to_string()),
+                ("query_script".to_string(), "cjk".to_string()),
+                ("rank".to_string(), "1".to_string()),
+            ]),
+            1,
+        )],
+        invocation_points(&metrics)?,
+    );
+    Ok(())
+}
+
+async fn start_turn(
+    registry: &ExtensionRegistry<()>,
+    session_store: &ExtensionData,
+    thread_store: &ExtensionData,
+    turn_id: &str,
+) -> Arc<ExtensionData> {
+    let turn_store = Arc::new(ExtensionData::new(turn_id));
+    registry.turn_input_contributors()[0]
+        .contribute(
+            TurnInputContext {
+                turn_id: turn_id.to_string(),
+                user_input: vec![UserInput::Text {
+                    text: "use demo".to_string(),
+                    text_elements: Vec::new(),
+                }],
+                environments: Vec::new(),
+            },
+            session_store,
+            thread_store,
+            turn_store.as_ref(),
+        )
+        .await;
+    turn_store
+}
+
+fn invocation_points(
+    metrics: &MetricsClient,
+) -> Result<Vec<InvocationPoint>, Box<dyn std::error::Error>> {
+    let snapshot = metrics.snapshot()?;
+    let metric = snapshot
+        .scope_metrics()
+        .flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics)
+        .find(|metric| metric.name() == INVOCATION_METRIC)
+        .ok_or("invocation metric was not emitted")?;
+    match metric.data() {
+        AggregatedMetrics::U64(MetricData::Sum(sum)) => Ok(sum
+            .data_points()
+            .map(|point| {
+                (
+                    point
+                        .attributes()
+                        .map(|attribute| {
+                            (
+                                attribute.key.as_str().to_string(),
+                                attribute.value.as_str().to_string(),
+                            )
+                        })
+                        .collect(),
+                    point.value(),
+                )
+            })
+            .collect()),
+        other => Err(format!("unexpected invocation metric data: {other:?}").into()),
+    }
+}

codex-rs/ext/skills/tests/skills_extension.rs

@@ -69,7 +69,8 @@ async fn installed_extension_uses_host_service_snapshot() -> TestResult {
             .ok_or("skill path should have a parent")?,
     )?;
     std::fs::write(&skill_path, DEMO_SKILL_CONTENTS)?;
-    let config = default_config();
+    let mut config = default_config();
+    config.shadow_selection_enabled = true;
 
     let mut builder = ExtensionRegistryBuilder::new();
     install(&mut builder, skills_extension_config);
@@ -781,6 +782,7 @@ struct TestConfig {
     include_instructions: bool,
     bundled_skills_enabled: bool,
     orchestrator_skills_enabled: bool,
+    shadow_selection_enabled: bool,
 }
 
 fn default_config() -> TestConfig {
@@ -788,6 +790,7 @@ fn default_config() -> TestConfig {
         include_instructions: true,
         bundled_skills_enabled: true,
         orchestrator_skills_enabled: true,
+        shadow_selection_enabled: false,
     }
 }
 
@@ -796,6 +799,7 @@ fn skills_extension_config(config: &TestConfig) -> SkillsExtensionConfig {
         include_instructions: config.include_instructions,
         bundled_skills_enabled: config.bundled_skills_enabled,
         orchestrator_skills_enabled: config.orchestrator_skills_enabled,
+        shadow_selection_enabled: config.shadow_selection_enabled,
     }
 }
 

codex-rs/features/src/lib.rs

@@ -206,6 +206,8 @@ pub enum Feature {
     ConcurrentReasoningSummaries,
     /// Allow prompting and installing missing MCP dependencies.
     SkillMcpDependencyInstall,
+    /// Run cheap skill-search methods in shadow mode and emit experiment metrics.
+    SkillSearch,
     /// Removed compatibility flag for deleted skill env var dependency prompting.
     SkillEnvVarDependencyPrompt,
     /// Enable the unified mention popup used by default in the TUI.
@@ -1190,6 +1192,12 @@ pub const FEATURES: &[FeatureSpec] = &[
         stage: Stage::Stable,
         default_enabled: true,
     },
+    FeatureSpec {
+        id: Feature::SkillSearch,
+        key: "skill_search",
+        stage: Stage::UnderDevelopment,
+        default_enabled: false,
+    },
     FeatureSpec {
         id: Feature::SkillEnvVarDependencyPrompt,
         key: "skill_env_var_dependency_prompt",