mirror of
https://github.com/openai/codex.git
synced 2026-11-17 14:58:36 +00:00
Add Guardian context cost and request token telemetry (#44164)
## What changed - Record per-section text bytes, estimated text tokens, image bytes, and image counts for synchronous reviews and asynchronous scoring without logging evidence payloads. - Emit estimated request tokens through `codex.guardian.context.request_tokens`. Synchronous estimates include assembled history, instructions, tool definitions, and output format, and measure the full logical request before WebSocket delta generation. Asynchronous estimates cover the assembled input. - Add shared context budgeting helpers, including conservative image token reservations independent of encoded payload size and model-aware input limit calculation. ## Testing Add coverage for separate text and image accounting, image estimates independent of encoded size, and section estimates that bound delivered messages. Extend asynchronous scorer and app-server tests to verify cost metric emission. GitOrigin-RevId: aed45ecd9c23706f88caa51f2a4f2c77872d3bdb
This commit is contained in:
1 parent
73a1148c9c
commit
d3ffbbed5a
13 files changed
+472
-9
No files matched your search
@@ -600,6 +600,21 @@ impl GuardianV2Extension {
|
||||
drop(history);
|
||||
drop(node_repl_images);
|
||||
truncations.extend(std::mem::take(&mut transcript.truncations));
|
||||
if let Some(metrics) = metrics.as_deref() {
|
||||
for (section, cost) in transcript.section_costs() {
|
||||
for (measurement, value) in cost.measurements() {
|
||||
metrics.histogram(
|
||||
codex_guardian_context::SECTION_COST_METRIC,
|
||||
i64::try_from(value).unwrap_or(i64::MAX),
|
||||
&[
|
||||
("target", "async"),
|
||||
("section", section),
|
||||
("measurement", measurement),
|
||||
],
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
let classification_input = transcript.into_messages();
|
||||
let mut failure_reason = "invalid_output";
|
||||
let mut classification_risk = None;
|
||||
|
||||
@@ -349,6 +349,14 @@ fn fast_decision_metric(decision: &str, reason: &str) -> RecordedMetric {
|
||||
#[derive(Default)]
|
||||
struct RecordingMetrics(Mutex<Vec<RecordedMetric>>);
|
||||
|
||||
impl RecordingMetrics {
|
||||
fn classification_samples(&self) -> Vec<RecordedMetric> {
|
||||
self.0.lock().unwrap().iter().filter(|sample| {
|
||||
!matches!(sample, RecordedMetric::Histogram(name, _, _) if name == codex_guardian_context::SECTION_COST_METRIC || name == codex_guardian_context::REQUEST_TOKENS_METRIC)
|
||||
}).cloned().collect()
|
||||
}
|
||||
}
|
||||
|
||||
impl ExtensionMetrics for RecordingMetrics {
|
||||
fn counter(&self, name: &str, inc: i64, tags: &[(&str, &str)]) {
|
||||
self.0.lock().unwrap().push(RecordedMetric::Counter(
|
||||
@@ -1436,12 +1444,22 @@ max_recent_non_user_entries = 8
|
||||
.latest_scored_tool_call
|
||||
.load(Ordering::Acquire)
|
||||
== 0
|
||||
|| metrics.0.lock().unwrap().len() < 14
|
||||
|| metrics.classification_samples().len() < 14
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
assert!(metrics.0.lock().unwrap().iter().any(|sample| {
|
||||
matches!(sample, RecordedMetric::Histogram(name, value, tags)
|
||||
if name == codex_guardian_context::SECTION_COST_METRIC
|
||||
&& *value > 0
|
||||
&& tags == &[
|
||||
("target".to_owned(), "async".to_owned()),
|
||||
("section".to_owned(), "conversation_transcript".to_owned()),
|
||||
("measurement".to_owned(), "text_bytes".to_owned()),
|
||||
])
|
||||
}));
|
||||
thread_store.insert(SecurityRiskScore {
|
||||
scores: BTreeMap::from([("action_risk".to_owned(), 0.65)]),
|
||||
call_id: None,
|
||||
@@ -1535,7 +1553,7 @@ max_recent_non_user_entries = 8
|
||||
Some(ReviewDecision::Approved)
|
||||
);
|
||||
|
||||
let samples = initial_metrics.0.lock().unwrap();
|
||||
let samples = initial_metrics.classification_samples();
|
||||
let classification_duration_ms = match &samples[9] {
|
||||
RecordedMetric::Histogram(name, duration_ms, _)
|
||||
if name == CLASSIFICATION_DURATION_METRIC =>
|
||||
@@ -1545,7 +1563,7 @@ max_recent_non_user_entries = 8
|
||||
sample => panic!("expected classification duration metric, got {sample:?}"),
|
||||
};
|
||||
assert_eq!(
|
||||
*samples,
|
||||
samples,
|
||||
[
|
||||
("total", 150),
|
||||
("input", 120),
|
||||
|
||||
@@ -292,6 +292,23 @@ impl LunaSampler {
|
||||
item.set_id(Some(ResponseItemId::new(prefix)));
|
||||
}
|
||||
}
|
||||
let total_tokens = input
|
||||
.iter()
|
||||
.map(codex_guardian_context::estimate_input_tokens)
|
||||
.fold(0usize, usize::saturating_add);
|
||||
if let Some(metrics) = self.config.metrics.as_deref() {
|
||||
for (component, tokens) in [
|
||||
("existing_context", 0),
|
||||
("new_input", total_tokens),
|
||||
("total", total_tokens),
|
||||
] {
|
||||
metrics.histogram(
|
||||
codex_guardian_context::REQUEST_TOKENS_METRIC,
|
||||
i64::try_from(tokens).unwrap_or(i64::MAX),
|
||||
&[("target", "async"), ("component", component)],
|
||||
);
|
||||
}
|
||||
}
|
||||
let mut request = ResponsesApiRequest {
|
||||
model: MODEL.to_owned(),
|
||||
instructions: String::new(),
|
||||
|
||||
@@ -276,14 +276,29 @@ async fn sampler_records_token_usage_after_returning_an_early_classification() -
|
||||
|
||||
assert_eq!(sampler.sample(sample_request("turn-1")).await?, "low");
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while metrics.0.lock().unwrap().len() < 7 {
|
||||
while metrics
|
||||
.0
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter(|metric| metric.0 == CLASSIFICATION_TOKEN_USAGE_METRIC)
|
||||
.count()
|
||||
< 7
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(
|
||||
*metrics.0.lock().unwrap(),
|
||||
metrics
|
||||
.0
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter(|metric| metric.0 == CLASSIFICATION_TOKEN_USAGE_METRIC)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>(),
|
||||
[
|
||||
("total", 37),
|
||||
("input", 37),
|
||||
@@ -300,6 +315,27 @@ async fn sampler_records_token_usage_after_returning_an_early_classification() -
|
||||
))
|
||||
);
|
||||
|
||||
let request = server
|
||||
.wait_for_request(
|
||||
/*connection_index*/ INITIAL_WEBSOCKET_CONNECTIONS - 1,
|
||||
/*request_index*/ 0,
|
||||
)
|
||||
.await
|
||||
.body_json();
|
||||
let input: Vec<ResponseItem> = serde_json::from_value(request["input"].clone())?;
|
||||
let estimated = input
|
||||
.iter()
|
||||
.map(codex_guardian_context::estimate_input_tokens)
|
||||
.sum::<usize>();
|
||||
assert!(metrics.0.lock().unwrap().contains(&(
|
||||
codex_guardian_context::REQUEST_TOKENS_METRIC.to_owned(),
|
||||
i64::try_from(estimated)?,
|
||||
vec![
|
||||
("target".to_owned(), "async".to_owned()),
|
||||
("component".to_owned(), "total".to_owned()),
|
||||
],
|
||||
)));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user