github/codex
4
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-19 16:17:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Record per-tool incremental update and description size metrics (#54133)

Browse Source 
## What changed

Tag `codex.tools.incremental_updates` counters with the qualified tool name alongside the action. Continue excluding initial catalogs and namespace headers, while counting each removed namespace member.

Add `codex.tools.spec_description_bytes` to measure UTF-8 description bytes when an existing tool definition is replaced. Skip definitions without descriptions and use logarithmic histogram boundaries up to 1 MiB.

## Testing

Extend namespace transition tests to check per-tool counts and description sizes. Add scenario assertions for replacement metrics with incremental tool updates and their absence on the non-incremental request path.

GitOrigin-RevId: e20e6d81f21a867b285df1f0da005c2806dd03c6
This commit is contained in:
Alex Kira
2026-10-14 23:55:50 +00:00
committed by copyberry
parent d292f72124
commit 348ef773ee
4 changed files with 201 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/core/src/context/world_state/top_level_tools.rs
+53 -47
View File
@@ -2,7 +2,7 @@
//! Missing state starts a fresh catalog; this section does not migrate legacy history.
//! One developer notice precedes each incremental namespace batch; catalog hashes are unchanged.
//! Whole namespace removals subsume their members in the removal notice.
//! Metrics count rendered tool redefinitions, excluding namespace headers and initial catalogs.
//! Metrics count rendered tool deltas and measure replacement description bytes.
use super::PreviousSectionState;
use super::SectionTransition;
@@ -13,6 +13,8 @@ use crate::context::ContextualUserFragment;
use crate::context::DeveloperInstructions;
use codex_extension_api::ExtensionMetrics;
use codex_otel::TOOL_INCREMENTAL_UPDATES_METRIC;
use codex_otel::TOOL_SPEC_DESCRIPTION_BYTES_METRIC;
use codex_otel::TOOL_SPEC_DESCRIPTION_BYTES_METRIC_BUCKETS;
use codex_prompts::ResolvedIncrementalToolMessages;
use codex_protocol::error::CodexErrorDetails;
use codex_protocol::error::Result;
@@ -89,47 +91,37 @@ fn definition_name(definition: &Value) -> &str {
.unwrap_or_else(|| definition["type"].as_str().expect("serialized tool type"))
}
/// Accumulates callable changes during rendering without retaining tool definitions.
#[derive(Default)]
struct ToolUpdateMetrics {
count_added: usize,
count_removed: usize,
count_schema_changed: usize,
}
fn record_changed_tool_metrics(
name: &str,
definition: &Value,
previous: Option<&BTreeMap<String, WorldStateHash>>,
metrics: Option<&dyn ExtensionMetrics>,
) {
const ACTION_TAG: &str = "action";
const TOOL_TAG: &str = "tool";
impl ToolUpdateMetrics {
fn count_changed_tool(
&mut self,
name: &str,
previous: Option<&BTreeMap<String, WorldStateHash>>,
) {
if let Some(previous) = previous {
if previous.contains_key(name) {
// The existing hash covers the entire definition, including descriptions.
self.count_schema_changed += 1;
} else {
self.count_added += 1;
}
}
}
fn record(self, metrics: Option<&dyn ExtensionMetrics>) {
let Some(metrics) = metrics else {
return;
};
for (action, count) in [
(ACTION_ADDED, self.count_added),
(ACTION_REMOVED, self.count_removed),
(ACTION_SCHEMA_CHANGED, self.count_schema_changed),
] {
if count > 0 {
metrics.counter(
TOOL_INCREMENTAL_UPDATES_METRIC,
i64::try_from(count).unwrap_or(i64::MAX),
&[("action", action)],
);
}
}
let (Some(previous), Some(metrics)) = (previous, metrics) else {
return;
};
let action = if previous.contains_key(name) {
ACTION_SCHEMA_CHANGED
} else {
ACTION_ADDED
};
metrics.counter(
TOOL_INCREMENTAL_UPDATES_METRIC,
/*inc*/ 1,
&[(ACTION_TAG, action), (TOOL_TAG, name)],
);
if action == ACTION_SCHEMA_CHANGED
&& let Some(description) = definition["description"].as_str()
{
metrics.histogram_with_boundaries(
TOOL_SPEC_DESCRIPTION_BYTES_METRIC,
i64::try_from(description.len()).unwrap_or(i64::MAX),
TOOL_SPEC_DESCRIPTION_BYTES_METRIC_BUCKETS.as_slice(),
&[(TOOL_TAG, name)],
);
}
}
@@ -147,14 +139,18 @@ impl WorldStateSection for TopLevelToolsState {
};
let changed =
|name: &str| previous.and_then(|previous| previous.get(name)) != self.hashes.get(name);
let mut counts = ToolUpdateMetrics::default();
let mut namespace_updates = Vec::new();
let mut tools = Vec::new();
for definition in &self.definitions {
let name = definition_name(definition);
let Some(members) = definition["tools"].as_array() else {
if changed(name) {
counts.count_changed_tool(name, previous);
record_changed_tool_metrics(
name,
definition,
previous,
self.metrics.as_deref(),
);
tools.push(definition.clone());
}
continue;
@@ -163,7 +159,12 @@ impl WorldStateSection for TopLevelToolsState {
for tool in members {
let tool_name = format!("{name}.{}", definition_name(tool));
if changed(&tool_name) {
counts.count_changed_tool(&tool_name, previous);
record_changed_tool_metrics(
&tool_name,
tool,
previous,
self.metrics.as_deref(),
);
changed_members.push(tool.clone());
}
}
@@ -229,9 +230,15 @@ impl WorldStateSection for TopLevelToolsState {
.keys()
.filter(|key| !self.hashes.contains_key(*key))
.filter(|key| !namespaces.contains(key.as_str()))
.inspect(|_| {
.inspect(|name| {
// Count members even when the notice collapses their removed namespace.
counts.count_removed += 1;
if let Some(metrics) = self.metrics.as_deref() {
metrics.counter(
TOOL_INCREMENTAL_UPDATES_METRIC,
/*inc*/ 1,
&[("action", ACTION_REMOVED), ("tool", name.as_str())],
);
}
})
.filter(|key| {
!key.split_once('.')
@@ -250,7 +257,6 @@ impl WorldStateSection for TopLevelToolsState {
);
}
}
counts.record(self.metrics.as_deref());
// Persist the empty map too: it is a known empty catalog, not missing state.
(Some(self.hashes.clone()), updates)
}
codex-rs/core/src/context/world_state/top_level_tools_tests.rs
+47 -7
View File
@@ -198,19 +198,59 @@ async fn snapshots_namespace_transitions() {
.attributes()
.map(|attribute| (attribute.key.as_str(), attribute.value.as_str().to_string()))
.collect::<BTreeMap<_, _>>();
assert_eq!(tags.len(), 1);
(tags["action"].clone(), point.value())
assert_eq!(tags.len(), 2);
(
(tags["action"].clone(), tags["tool"].clone()),
point.value(),
)
})
.collect::<BTreeMap<_, _>>();
// Initial/full snapshots and unchanged catalogs contribute nothing; namespace
// headers do not count, and removing a whole namespace counts each member.
assert_eq!(
counts,
BTreeMap::from([
("added".to_string(), 6),
("removed".to_string(), 11),
("schema_changed".to_string(), 1),
])
[
("added", "functions.added", 1),
("added", "functions.changed", 1),
("added", "functions.removed", 1),
("added", "functions.restored", 1),
("added", "functions.unchanged", 2),
("removed", "functions.added", 3),
("removed", "functions.changed", 3),
("removed", "functions.removed", 1),
("removed", "functions.unchanged", 2),
("removed", "other.lookup", 2),
("schema_changed", "functions.changed", 1),
]
.into_iter()
.map(|(action, tool, count)| ((action.to_string(), tool.to_string()), count))
.collect()
);
let histogram = snapshot
.scope_metrics()
.flat_map(ScopeMetrics::metrics)
.find(|metric| metric.name() == TOOL_SPEC_DESCRIPTION_BYTES_METRIC)
.expect("tool description byte histogram");
let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = histogram.data() else {
panic!("tool description bytes must be a histogram");
};
assert_eq!(
histogram
.data_points()
.map(|point| (
point
.attributes()
.map(|tag| (tag.key.as_str(), tag.value.to_string()))
.collect::<Vec<_>>(),
point.count(),
point.sum(),
))
.collect::<Vec<_>>(),
vec![(
vec![("tool", "functions.changed".to_string())],
1,
"Look up a value.".len() as f64
)],
);
metrics.shutdown().unwrap();
}
codex-rs/core/tests/suite/scenarios_incremental_tools.rs
+94 -1
View File
@@ -2,6 +2,10 @@
use anyhow::Result;
use codex_features::Feature;
use codex_otel::MetricsClient;
use codex_otel::MetricsConfig;
use codex_otel::TOOL_INCREMENTAL_UPDATES_METRIC;
use codex_otel::TOOL_SPEC_DESCRIPTION_BYTES_METRIC;
use codex_protocol::openai_models::CodeModeToolMessages;
use codex_protocol::openai_models::IncrementalToolMessages;
use codex_protocol::openai_models::ToolMessage;
@@ -17,8 +21,13 @@ use core_test_support::skip_if_wine_exec;
use core_test_support::test_codex::test_codex;
use core_test_support::wait_for_event;
use core_test_support::wait_for_mcp_server;
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
use opentelemetry_sdk::metrics::data::AggregatedMetrics;
use opentelemetry_sdk::metrics::data::MetricData;
use opentelemetry_sdk::metrics::data::ScopeMetrics;
use pretty_assertions::assert_eq;
use serde_json::json;
use std::collections::BTreeMap;
use test_case::test_case;
use super::super::rmcp_client::remote_aware_environment_id;
@@ -52,7 +61,16 @@ async fn incremental_tools_append_changed_catalog_without_rewriting_history(
.collect(),
)
.await;
let test = test_codex()
let metrics = MetricsClient::new(
MetricsConfig::in_memory(
"test",
"codex-core",
env!("CARGO_PKG_VERSION"),
InMemoryMetricExporter::default(),
)
.with_runtime_reader(),
)?;
let mut test = test_codex()
.with_model_info_override("gpt-5.4", move |model| {
model.use_responses_lite = use_responses_lite;
model.tool_mode = Some(ToolMode::CodeMode);
@@ -106,6 +124,12 @@ async fn incremental_tools_append_changed_catalog_without_rewriting_history(
})
.build_with_auto_env(&server)
.await?;
let mut options = test.start_thread_options().await;
options.thread_extension_init.insert(metrics.clone());
let thread = test.thread_manager.start_thread(options).await?;
test.codex.shutdown_and_wait().await?;
test.codex = thread.thread;
test.session_configured = thread.session_configured;
test.submit_turn("Begin work.").await?;
test.submit_turn("Continue with the same tools.").await?;
core_test_support::submit_thread_settings(
@@ -130,6 +154,74 @@ async fn incremental_tools_append_changed_catalog_without_rewriting_history(
}
let requests = mock.requests();
assert_eq!(requests.len(), if use_responses_lite { 6 } else { 4 });
let snapshot = metrics.snapshot()?;
let mut counts = BTreeMap::<_, u64>::new();
let mut bytes = BTreeMap::new();
for metric in snapshot.scope_metrics().flat_map(ScopeMetrics::metrics) {
match metric.name() {
TOOL_INCREMENTAL_UPDATES_METRIC => {
let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else {
panic!("tool updates must be a counter");
};
for point in sum.data_points() {
let tags = point
.attributes()
.map(|tag| (tag.key.as_str(), tag.value.to_string()))
.collect::<BTreeMap<_, _>>();
*counts
.entry((tags["action"].clone(), tags["tool"].clone()))
.or_default() += point.value();
}
}
TOOL_SPEC_DESCRIPTION_BYTES_METRIC => {
let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else {
panic!("tool description bytes must be a histogram");
};
for point in histogram.data_points() {
let tool = point
.attributes()
.find(|tag| tag.key.as_str() == "tool")
.expect("tool tag")
.value
.to_string();
let sample = bytes.entry(tool).or_insert((0, 0.0));
sample.0 += point.count();
sample.1 += point.sum();
}
}
_ => {}
}
}
if use_responses_lite {
let body = requests[2].body_json();
let exec = body["input"]
.as_array()
.expect("request input")
.iter()
.rev()
.filter(|item| item["type"] == "additional_tools")
.find_map(|item| responses::namespace_child_tool(item, "functions", "exec"))
.expect("changed exec definition");
let description_bytes = exec["description"]
.as_str()
.expect("exec description")
.len() as f64;
assert_eq!(
counts,
BTreeMap::from([(
("schema_changed".to_string(), "functions.exec".to_string()),
1
)])
);
assert_eq!(
bytes,
BTreeMap::from([("functions.exec".to_string(), (1, description_bytes))])
);
} else {
assert_eq!(counts, BTreeMap::new());
assert_eq!(bytes, BTreeMap::new());
}
if use_responses_lite {
for request in &requests[..5] {
assert_eq!(
@@ -161,6 +253,7 @@ async fn incremental_tools_append_changed_catalog_without_rewriting_history(
.include_request_settings(),
)
);
metrics.shutdown()?;
Ok(())
}
codex-rs/otel/src/metrics/names.rs
+7 -1
View File
@@ -90,10 +90,16 @@ pub const TOOL_REGISTRATIONS_METRIC_BUCKETS: &[f64] = &[
pub static THREAD_TOOLS_METRIC_BUCKETS: LazyLock<[f64; 511]> =
LazyLock::new(|| context_log_buckets(/*max_exponent*/ 15.0));
/// Rendered incremental tool deltas, tagged by action; excludes initial catalogs.
/// Rendered incremental tool deltas by action and qualified tool; excludes initial catalogs.
/// schema_changed includes any definition change, such as description edits.
pub const TOOL_INCREMENTAL_UPDATES_METRIC: &str = "codex.tools.incremental_updates";
/// UTF-8 description bytes on replacement by qualified tool; skips missing descriptions.
pub const TOOL_SPEC_DESCRIPTION_BYTES_METRIC: &str = "codex.tools.spec_description_bytes";
/// Logarithmic boundaries up to 1 MiB.
pub static TOOL_SPEC_DESCRIPTION_BYTES_METRIC_BUCKETS: LazyLock<[f64; 511]> =
LazyLock::new(|| context_log_buckets(/*max_exponent*/ 20.0));
pub static THREAD_SKILLS_COUNT_METRIC_BUCKETS: LazyLock<[f64; 513]> =
LazyLock::new(|| std::array::from_fn(|index| index as f64));