mirror of
https://github.com/openai/codex.git
synced 2026-11-13 14:46:14 +00:00
Record metrics for empty skill catalogs (#35607)
## Why Empty rendered skill catalogs skipped catalog telemetry instead of reporting zero-valued counts. ## What changed - Record a default render report when thread, turn-input, or executor catalog rendering produces no fragment. - Record zero-valued host world-state metrics when skill instructions are enabled but no host skills are available. - Preserve the `catalog_surface` tag so empty host and executor catalogs remain distinguishable. ## Testing Add coverage for empty catalog rendering and mixed nonempty-executor, empty-host world state. GitOrigin-RevId: f68d65806703c0ded1ece1e5ae1ffaf6d6045126
This commit is contained in:
1 parent
d6ea5991e7
commit
16d7daad7c
5 files changed
+243
-27
No files matched your search
@@ -46,6 +46,7 @@ use crate::render::MAX_SKILL_NAME_BYTES;
|
||||
use crate::render::MAX_SKILL_PATH_BYTES;
|
||||
use crate::render::SkillCatalogRenderPolicy;
|
||||
use crate::render::SkillMetadataBudget;
|
||||
use crate::render::SkillRenderReport;
|
||||
use crate::render::capped_skill_metadata_budget;
|
||||
use crate::render::render_available_skills;
|
||||
use crate::render::truncate_main_prompt_contents;
|
||||
@@ -87,6 +88,12 @@ fn render_catalog(
|
||||
budget: SkillMetadataBudget,
|
||||
) -> RenderedCatalog {
|
||||
let Some(rendered) = render_available_skills(catalog, policy, budget) else {
|
||||
record_catalog_render(
|
||||
extension_metrics,
|
||||
catalog_surface,
|
||||
budget,
|
||||
&SkillRenderReport::default(),
|
||||
);
|
||||
return RenderedCatalog::default();
|
||||
};
|
||||
record_catalog_render(extension_metrics, catalog_surface, budget, &rendered.report);
|
||||
@@ -98,6 +105,10 @@ fn render_catalog(
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "extension_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
impl<C> ThreadLifecycleContributor<C> for SkillsExtension<C>
|
||||
where
|
||||
C: Send + Sync + 'static,
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
use std::sync::Mutex;
|
||||
|
||||
use codex_extension_api::ExtensionMetrics;
|
||||
use codex_otel::THREAD_SKILLS_DESCRIPTION_TRUNCATED_CHARS_METRIC;
|
||||
use codex_otel::THREAD_SKILLS_ENABLED_TOTAL_METRIC;
|
||||
use codex_otel::THREAD_SKILLS_KEPT_TOTAL_METRIC;
|
||||
use codex_otel::THREAD_SKILLS_TRUNCATED_METRIC;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[derive(Default)]
|
||||
struct RecordingMetrics {
|
||||
samples: Mutex<Vec<(String, i64)>>,
|
||||
}
|
||||
|
||||
impl ExtensionMetrics for RecordingMetrics {
|
||||
fn histogram(&self, name: &str, value: i64, _tags: &[(&str, &str)]) {
|
||||
self.samples
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.push((name.to_string(), value));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_catalog_records_zero_metrics_without_a_fragment() {
|
||||
let metrics = RecordingMetrics::default();
|
||||
|
||||
let rendered = render_catalog(
|
||||
Some(&metrics),
|
||||
CatalogSurface::ThreadContext,
|
||||
&SkillCatalog::default(),
|
||||
/*include_skills_usage_instructions*/ false,
|
||||
SkillCatalogRenderPolicy::ExtensionCompatible,
|
||||
SkillMetadataBudget::Characters(8_000),
|
||||
);
|
||||
|
||||
assert!(rendered.fragment.is_none());
|
||||
assert_eq!(rendered.warning_message, None);
|
||||
|
||||
assert_eq!(
|
||||
*metrics
|
||||
.samples
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner),
|
||||
vec![
|
||||
(THREAD_SKILLS_ENABLED_TOTAL_METRIC.to_string(), 0),
|
||||
(THREAD_SKILLS_KEPT_TOTAL_METRIC.to_string(), 0),
|
||||
(THREAD_SKILLS_TRUNCATED_METRIC.to_string(), 0),
|
||||
(
|
||||
THREAD_SKILLS_DESCRIPTION_TRUNCATED_CHARS_METRIC.to_string(),
|
||||
0,
|
||||
),
|
||||
]
|
||||
);
|
||||
}
|
||||
@@ -88,7 +88,7 @@ pub(crate) enum SkillMetadataBudget {
|
||||
Characters(usize),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct SkillRenderReport {
|
||||
pub(crate) total_count: usize,
|
||||
pub(crate) included_count: usize,
|
||||
|
||||
@@ -97,7 +97,20 @@ pub(crate) fn host_skills_world_state_section(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let render_report = available.as_ref().map(|available| available.report.clone());
|
||||
let render_metrics = include_instructions.then(|| {
|
||||
available
|
||||
.as_ref()
|
||||
.map(|available| {
|
||||
let report = &available.report;
|
||||
(
|
||||
report.total_count,
|
||||
report.included_count,
|
||||
report.omitted_count,
|
||||
report.truncated_description_chars,
|
||||
)
|
||||
})
|
||||
.unwrap_or_default()
|
||||
});
|
||||
let body = available.map(|available| {
|
||||
AvailableSkillsInstructions::from_available_skills(
|
||||
available,
|
||||
@@ -128,14 +141,16 @@ pub(crate) fn host_skills_world_state_section(
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(report) = &render_report {
|
||||
if let Some((total_count, included_count, omitted_count, truncated_description_chars)) =
|
||||
render_metrics
|
||||
{
|
||||
record_catalog_metrics(
|
||||
extension_metrics.as_deref(),
|
||||
CatalogSurface::HostWorldState,
|
||||
report.total_count,
|
||||
report.included_count,
|
||||
report.omitted_count,
|
||||
report.truncated_description_chars,
|
||||
total_count,
|
||||
included_count,
|
||||
omitted_count,
|
||||
truncated_description_chars,
|
||||
);
|
||||
}
|
||||
let body = match body.as_deref() {
|
||||
|
||||
Reference in new issue
Block a user