Record metrics for empty skill catalogs (#35607)

## Why

Empty rendered skill catalogs skipped catalog telemetry instead of reporting
zero-valued counts.

## What changed

- Record a default render report when thread, turn-input, or executor catalog
  rendering produces no fragment.
- Record zero-valued host world-state metrics when skill instructions are
  enabled but no host skills are available.
- Preserve the `catalog_surface` tag so empty host and executor catalogs remain
  distinguishable.

## Testing

Add coverage for empty catalog rendering and mixed nonempty-executor,
empty-host world state.

GitOrigin-RevId: f68d65806703c0ded1ece1e5ae1ffaf6d6045126
This commit is contained in:
felixxia-oai authored and copyberry committed 2026-07-27 12:32:32 +00:00
1 parent d6ea5991e7
commit 16d7daad7c
5 files changed
+243 -27

No files matched your search

+11
View File
@@ -46,6 +46,7 @@ use crate::render::MAX_SKILL_NAME_BYTES;
use crate::render::MAX_SKILL_PATH_BYTES;
use crate::render::SkillCatalogRenderPolicy;
use crate::render::SkillMetadataBudget;
use crate::render::SkillRenderReport;
use crate::render::capped_skill_metadata_budget;
use crate::render::render_available_skills;
use crate::render::truncate_main_prompt_contents;
@@ -87,6 +88,12 @@ fn render_catalog(
budget: SkillMetadataBudget,
) -> RenderedCatalog {
let Some(rendered) = render_available_skills(catalog, policy, budget) else {
record_catalog_render(
extension_metrics,
catalog_surface,
budget,
&SkillRenderReport::default(),
);
return RenderedCatalog::default();
};
record_catalog_render(extension_metrics, catalog_surface, budget, &rendered.report);
@@ -98,6 +105,10 @@ fn render_catalog(
}
}
#[cfg(test)]
#[path = "extension_tests.rs"]
mod tests;
impl<C> ThreadLifecycleContributor<C> for SkillsExtension<C>
where
C: Send + Sync + 'static,
@@ -0,0 +1,57 @@
use std::sync::Mutex;
use codex_extension_api::ExtensionMetrics;
use codex_otel::THREAD_SKILLS_DESCRIPTION_TRUNCATED_CHARS_METRIC;
use codex_otel::THREAD_SKILLS_ENABLED_TOTAL_METRIC;
use codex_otel::THREAD_SKILLS_KEPT_TOTAL_METRIC;
use codex_otel::THREAD_SKILLS_TRUNCATED_METRIC;
use pretty_assertions::assert_eq;
use super::*;
#[derive(Default)]
struct RecordingMetrics {
samples: Mutex<Vec<(String, i64)>>,
}
impl ExtensionMetrics for RecordingMetrics {
fn histogram(&self, name: &str, value: i64, _tags: &[(&str, &str)]) {
self.samples
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push((name.to_string(), value));
}
}
#[test]
fn empty_catalog_records_zero_metrics_without_a_fragment() {
let metrics = RecordingMetrics::default();
let rendered = render_catalog(
Some(&metrics),
CatalogSurface::ThreadContext,
&SkillCatalog::default(),
/*include_skills_usage_instructions*/ false,
SkillCatalogRenderPolicy::ExtensionCompatible,
SkillMetadataBudget::Characters(8_000),
);
assert!(rendered.fragment.is_none());
assert_eq!(rendered.warning_message, None);
assert_eq!(
*metrics
.samples
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
vec![
(THREAD_SKILLS_ENABLED_TOTAL_METRIC.to_string(), 0),
(THREAD_SKILLS_KEPT_TOTAL_METRIC.to_string(), 0),
(THREAD_SKILLS_TRUNCATED_METRIC.to_string(), 0),
(
THREAD_SKILLS_DESCRIPTION_TRUNCATED_CHARS_METRIC.to_string(),
0,
),
]
);
}
+1 -1
View File
@@ -88,7 +88,7 @@ pub(crate) enum SkillMetadataBudget {
Characters(usize),
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub(crate) struct SkillRenderReport {
pub(crate) total_count: usize,
pub(crate) included_count: usize,
+21 -6
View File
@@ -97,7 +97,20 @@ pub(crate) fn host_skills_world_state_section(
} else {
None
};
let render_report = available.as_ref().map(|available| available.report.clone());
let render_metrics = include_instructions.then(|| {
available
.as_ref()
.map(|available| {
let report = &available.report;
(
report.total_count,
report.included_count,
report.omitted_count,
report.truncated_description_chars,
)
})
.unwrap_or_default()
});
let body = available.map(|available| {
AvailableSkillsInstructions::from_available_skills(
available,
@@ -128,14 +141,16 @@ pub(crate) fn host_skills_world_state_section(
}
}
if let Some(report) = &render_report {
if let Some((total_count, included_count, omitted_count, truncated_description_chars)) =
render_metrics
{
record_catalog_metrics(
extension_metrics.as_deref(),
CatalogSurface::HostWorldState,
report.total_count,
report.included_count,
report.omitted_count,
report.truncated_description_chars,
total_count,
included_count,
omitted_count,
truncated_description_chars,
);
}
let body = match body.as_deref() {