[codex] Trace cancelled inference streams (#19839)

Records cancelled inference streams when Codex stops consuming a
provider response before `response.completed`, preserving complete
output items observed before cancellation.

Also closes still-running inference calls when the owning turn ends, so
reduced rollout traces do not leave stale `Running` inference nodes.

Covered by focused reducer coverage and a core stream-drop test for
partial output preservation.
This commit is contained in:
cassirer-openai authored and GitHub committed 2026-04-27 21:58:29 +00:00
1 parent 2be9fd5a93
commit 30c5c768de
10 files changed
+556 -39

No files matched your search

@@ -103,6 +103,35 @@ impl TraceReducer {
Ok(())
}
/// Closes any inference streams that are still live when the owning turn ends.
///
/// Normal completion events close the active inference before the turn ends.
/// If a call is still `Running`, Codex stopped observing that provider stream
/// earlier and the reduced graph should not present it as live.
pub(super) fn close_running_inference_calls_for_turn_end(
&mut self,
seq: RawEventSeq,
wall_time_unix_ms: i64,
codex_turn_id: &str,
turn_status: &ExecutionStatus,
) {
let inference_status = match turn_status {
ExecutionStatus::Running => return,
ExecutionStatus::Completed | ExecutionStatus::Cancelled => ExecutionStatus::Cancelled,
ExecutionStatus::Failed => ExecutionStatus::Failed,
ExecutionStatus::Aborted => ExecutionStatus::Aborted,
};
for inference in self.rollout.inference_calls.values_mut() {
if inference.codex_turn_id == codex_turn_id
&& inference.execution.status == ExecutionStatus::Running
{
inference.execution.ended_at_unix_ms = Some(wall_time_unix_ms);
inference.execution.ended_seq = Some(seq);
inference.execution.status = inference_status.clone();
}
}
}
/// Completes an inference call and, when present, reduces response output items.
pub(super) fn complete_inference_call(
&mut self,
@@ -130,14 +159,25 @@ impl TraceReducer {
let Some(inference) = self.rollout.inference_calls.get_mut(&inference_call_id) else {
bail!("inference call {inference_call_id} disappeared during response reduction");
};
inference.execution.ended_at_unix_ms = Some(wall_time_unix_ms);
inference.execution.ended_seq = Some(seq);
inference.execution.status = status;
inference.upstream_request_id = response_id;
inference.raw_response_payload_id = response_payload.map(|payload| payload.raw_payload_id);
// Turn-end cleanup can close a stream before the async mapper observes
// cancellation. Preserve that terminal status while still retaining any
// late partial response evidence from the mapper.
if inference.execution.status == ExecutionStatus::Running {
inference.execution.ended_at_unix_ms = Some(wall_time_unix_ms);
inference.execution.ended_seq = Some(seq);
inference.execution.status = status;
inference.upstream_request_id = response_id;
}
if let Some(response_payload) = response_payload {
inference.raw_response_payload_id = Some(response_payload.raw_payload_id);
}
if let Some(response_item_ids) = response_item_ids {
inference.response_item_ids = response_item_ids;
}
Ok(())
}
}
#[cfg(test)]
#[path = "inference_tests.rs"]
mod tests;
@@ -0,0 +1,148 @@
use pretty_assertions::assert_eq;
use serde_json::json;
use tempfile::TempDir;
use crate::model::ConversationItemKind;
use crate::model::ExecutionStatus;
use crate::payload::RawPayloadKind;
use crate::raw_event::RawTraceEventPayload;
use crate::reducer::test_support::append_inference_start;
use crate::reducer::test_support::create_started_writer;
use crate::reducer::test_support::message;
use crate::reducer::test_support::start_turn;
use crate::replay_bundle;
#[test]
fn cancelled_inference_reduces_partial_response_items() -> anyhow::Result<()> {
let temp = TempDir::new()?;
let writer = create_started_writer(&temp)?;
start_turn(&writer, "turn-1")?;
let request = writer.write_json_payload(
RawPayloadKind::InferenceRequest,
&json!({
"input": [message("user", "draft")]
}),
)?;
append_inference_start(&writer, "inference-1", "turn-1", request)?;
let partial_response = writer.write_json_payload(
RawPayloadKind::InferenceResponse,
&json!({
"response_id": null,
"token_usage": null,
"output_items": [{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "partial"}]
}]
}),
)?;
writer.append(RawTraceEventPayload::InferenceCancelled {
inference_call_id: "inference-1".to_string(),
reason: "test interruption".to_string(),
partial_response_payload: Some(partial_response),
})?;
let rollout = replay_bundle(temp.path())?;
let inference = &rollout.inference_calls["inference-1"];
let response_item_id = &inference.response_item_ids[0];
assert_eq!(inference.execution.status, ExecutionStatus::Cancelled);
assert_eq!(inference.response_item_ids.len(), 1);
assert_eq!(
rollout.conversation_items[response_item_id].kind,
ConversationItemKind::Message,
);
assert_eq!(
rollout.conversation_items[response_item_id].produced_by,
vec![crate::model::ProducerRef::Inference {
inference_call_id: "inference-1".to_string(),
}],
);
Ok(())
}
#[test]
fn cancelled_turn_closes_running_inference_call() -> anyhow::Result<()> {
let temp = TempDir::new()?;
let writer = create_started_writer(&temp)?;
start_turn(&writer, "turn-1")?;
let request = writer.write_json_payload(
RawPayloadKind::InferenceRequest,
&json!({
"input": [message("user", "wait")]
}),
)?;
append_inference_start(&writer, "inference-1", "turn-1", request)?;
let turn_end = writer.append(RawTraceEventPayload::CodexTurnEnded {
codex_turn_id: "turn-1".to_string(),
status: ExecutionStatus::Cancelled,
})?;
let rollout = replay_bundle(temp.path())?;
let inference = &rollout.inference_calls["inference-1"];
assert_eq!(inference.execution.status, ExecutionStatus::Cancelled);
assert_eq!(inference.execution.ended_seq, Some(turn_end.seq));
Ok(())
}
#[test]
fn late_cancelled_inference_preserves_turn_end_status() -> anyhow::Result<()> {
let temp = TempDir::new()?;
let writer = create_started_writer(&temp)?;
start_turn(&writer, "turn-1")?;
let request = writer.write_json_payload(
RawPayloadKind::InferenceRequest,
&json!({
"input": [message("user", "interrupt")]
}),
)?;
append_inference_start(&writer, "inference-1", "turn-1", request)?;
let turn_end = writer.append(RawTraceEventPayload::CodexTurnEnded {
codex_turn_id: "turn-1".to_string(),
status: ExecutionStatus::Failed,
})?;
let partial_response = writer.write_json_payload(
RawPayloadKind::InferenceResponse,
&json!({
"response_id": null,
"token_usage": null,
"output_items": [{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "late partial"}]
}]
}),
)?;
writer.append(RawTraceEventPayload::InferenceCancelled {
inference_call_id: "inference-1".to_string(),
reason: "stream mapper noticed cancellation after turn end".to_string(),
partial_response_payload: Some(partial_response.clone()),
})?;
let rollout = replay_bundle(temp.path())?;
let inference = &rollout.inference_calls["inference-1"];
assert_eq!(inference.execution.status, ExecutionStatus::Failed);
assert_eq!(inference.execution.ended_seq, Some(turn_end.seq));
assert_eq!(
inference.raw_response_payload_id,
Some(partial_response.raw_payload_id),
);
assert_eq!(inference.response_item_ids.len(), 1);
let response_item_id = &inference.response_item_ids[0];
assert_eq!(
rollout.conversation_items[response_item_id].body.parts,
vec![crate::model::ConversationPart::Text {
text: "late partial".to_string(),
}],
);
Ok(())
}
+14
View File
@@ -254,6 +254,20 @@ impl TraceReducer {
partial_response_payload,
)?;
}
RawTraceEventPayload::InferenceCancelled {
inference_call_id,
partial_response_payload,
..
} => {
self.complete_inference_call(
event.seq,
event.wall_time_unix_ms,
inference_call_id,
ExecutionStatus::Cancelled,
/*response_id*/ None,
partial_response_payload,
)?;
}
RawTraceEventPayload::ProtocolEventObserved { .. } => {
// Protocol wrappers are raw debug breadcrumbs. Typed hooks own
// the reduced graph, so these payload refs are retained without
@@ -187,6 +187,12 @@ impl TraceReducer {
&codex_turn_id,
&status,
)?;
self.close_running_inference_calls_for_turn_end(
seq,
wall_time_unix_ms,
&codex_turn_id,
&status,
);
Ok(())
}