mirror of
https://github.com/openai/codex.git
synced 2026-11-17 14:58:36 +00:00
[codex] Trace cancelled inference streams (#19839)
Records cancelled inference streams when Codex stops consuming a provider response before `response.completed`, preserving complete output items observed before cancellation. Also closes still-running inference calls when the owning turn ends, so reduced rollout traces do not leave stale `Running` inference nodes. Covered by focused reducer coverage and a core stream-drop test for partial output preservation.
This commit is contained in:
1 parent
2be9fd5a93
commit
30c5c768de
10 files changed
+556
-39
No files matched your search
@@ -103,6 +103,35 @@ impl TraceReducer {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Closes any inference streams that are still live when the owning turn ends.
|
||||
///
|
||||
/// Normal completion events close the active inference before the turn ends.
|
||||
/// If a call is still `Running`, Codex stopped observing that provider stream
|
||||
/// earlier and the reduced graph should not present it as live.
|
||||
pub(super) fn close_running_inference_calls_for_turn_end(
|
||||
&mut self,
|
||||
seq: RawEventSeq,
|
||||
wall_time_unix_ms: i64,
|
||||
codex_turn_id: &str,
|
||||
turn_status: &ExecutionStatus,
|
||||
) {
|
||||
let inference_status = match turn_status {
|
||||
ExecutionStatus::Running => return,
|
||||
ExecutionStatus::Completed | ExecutionStatus::Cancelled => ExecutionStatus::Cancelled,
|
||||
ExecutionStatus::Failed => ExecutionStatus::Failed,
|
||||
ExecutionStatus::Aborted => ExecutionStatus::Aborted,
|
||||
};
|
||||
for inference in self.rollout.inference_calls.values_mut() {
|
||||
if inference.codex_turn_id == codex_turn_id
|
||||
&& inference.execution.status == ExecutionStatus::Running
|
||||
{
|
||||
inference.execution.ended_at_unix_ms = Some(wall_time_unix_ms);
|
||||
inference.execution.ended_seq = Some(seq);
|
||||
inference.execution.status = inference_status.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Completes an inference call and, when present, reduces response output items.
|
||||
pub(super) fn complete_inference_call(
|
||||
&mut self,
|
||||
@@ -130,14 +159,25 @@ impl TraceReducer {
|
||||
let Some(inference) = self.rollout.inference_calls.get_mut(&inference_call_id) else {
|
||||
bail!("inference call {inference_call_id} disappeared during response reduction");
|
||||
};
|
||||
inference.execution.ended_at_unix_ms = Some(wall_time_unix_ms);
|
||||
inference.execution.ended_seq = Some(seq);
|
||||
inference.execution.status = status;
|
||||
inference.upstream_request_id = response_id;
|
||||
inference.raw_response_payload_id = response_payload.map(|payload| payload.raw_payload_id);
|
||||
// Turn-end cleanup can close a stream before the async mapper observes
|
||||
// cancellation. Preserve that terminal status while still retaining any
|
||||
// late partial response evidence from the mapper.
|
||||
if inference.execution.status == ExecutionStatus::Running {
|
||||
inference.execution.ended_at_unix_ms = Some(wall_time_unix_ms);
|
||||
inference.execution.ended_seq = Some(seq);
|
||||
inference.execution.status = status;
|
||||
inference.upstream_request_id = response_id;
|
||||
}
|
||||
if let Some(response_payload) = response_payload {
|
||||
inference.raw_response_payload_id = Some(response_payload.raw_payload_id);
|
||||
}
|
||||
if let Some(response_item_ids) = response_item_ids {
|
||||
inference.response_item_ids = response_item_ids;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "inference_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,148 @@
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
use tempfile::TempDir;
|
||||
|
||||
use crate::model::ConversationItemKind;
|
||||
use crate::model::ExecutionStatus;
|
||||
use crate::payload::RawPayloadKind;
|
||||
use crate::raw_event::RawTraceEventPayload;
|
||||
use crate::reducer::test_support::append_inference_start;
|
||||
use crate::reducer::test_support::create_started_writer;
|
||||
use crate::reducer::test_support::message;
|
||||
use crate::reducer::test_support::start_turn;
|
||||
use crate::replay_bundle;
|
||||
|
||||
#[test]
|
||||
fn cancelled_inference_reduces_partial_response_items() -> anyhow::Result<()> {
|
||||
let temp = TempDir::new()?;
|
||||
let writer = create_started_writer(&temp)?;
|
||||
start_turn(&writer, "turn-1")?;
|
||||
|
||||
let request = writer.write_json_payload(
|
||||
RawPayloadKind::InferenceRequest,
|
||||
&json!({
|
||||
"input": [message("user", "draft")]
|
||||
}),
|
||||
)?;
|
||||
append_inference_start(&writer, "inference-1", "turn-1", request)?;
|
||||
|
||||
let partial_response = writer.write_json_payload(
|
||||
RawPayloadKind::InferenceResponse,
|
||||
&json!({
|
||||
"response_id": null,
|
||||
"token_usage": null,
|
||||
"output_items": [{
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "partial"}]
|
||||
}]
|
||||
}),
|
||||
)?;
|
||||
writer.append(RawTraceEventPayload::InferenceCancelled {
|
||||
inference_call_id: "inference-1".to_string(),
|
||||
reason: "test interruption".to_string(),
|
||||
partial_response_payload: Some(partial_response),
|
||||
})?;
|
||||
|
||||
let rollout = replay_bundle(temp.path())?;
|
||||
let inference = &rollout.inference_calls["inference-1"];
|
||||
let response_item_id = &inference.response_item_ids[0];
|
||||
|
||||
assert_eq!(inference.execution.status, ExecutionStatus::Cancelled);
|
||||
assert_eq!(inference.response_item_ids.len(), 1);
|
||||
assert_eq!(
|
||||
rollout.conversation_items[response_item_id].kind,
|
||||
ConversationItemKind::Message,
|
||||
);
|
||||
assert_eq!(
|
||||
rollout.conversation_items[response_item_id].produced_by,
|
||||
vec![crate::model::ProducerRef::Inference {
|
||||
inference_call_id: "inference-1".to_string(),
|
||||
}],
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancelled_turn_closes_running_inference_call() -> anyhow::Result<()> {
|
||||
let temp = TempDir::new()?;
|
||||
let writer = create_started_writer(&temp)?;
|
||||
start_turn(&writer, "turn-1")?;
|
||||
|
||||
let request = writer.write_json_payload(
|
||||
RawPayloadKind::InferenceRequest,
|
||||
&json!({
|
||||
"input": [message("user", "wait")]
|
||||
}),
|
||||
)?;
|
||||
append_inference_start(&writer, "inference-1", "turn-1", request)?;
|
||||
let turn_end = writer.append(RawTraceEventPayload::CodexTurnEnded {
|
||||
codex_turn_id: "turn-1".to_string(),
|
||||
status: ExecutionStatus::Cancelled,
|
||||
})?;
|
||||
|
||||
let rollout = replay_bundle(temp.path())?;
|
||||
let inference = &rollout.inference_calls["inference-1"];
|
||||
|
||||
assert_eq!(inference.execution.status, ExecutionStatus::Cancelled);
|
||||
assert_eq!(inference.execution.ended_seq, Some(turn_end.seq));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn late_cancelled_inference_preserves_turn_end_status() -> anyhow::Result<()> {
|
||||
let temp = TempDir::new()?;
|
||||
let writer = create_started_writer(&temp)?;
|
||||
start_turn(&writer, "turn-1")?;
|
||||
|
||||
let request = writer.write_json_payload(
|
||||
RawPayloadKind::InferenceRequest,
|
||||
&json!({
|
||||
"input": [message("user", "interrupt")]
|
||||
}),
|
||||
)?;
|
||||
append_inference_start(&writer, "inference-1", "turn-1", request)?;
|
||||
let turn_end = writer.append(RawTraceEventPayload::CodexTurnEnded {
|
||||
codex_turn_id: "turn-1".to_string(),
|
||||
status: ExecutionStatus::Failed,
|
||||
})?;
|
||||
|
||||
let partial_response = writer.write_json_payload(
|
||||
RawPayloadKind::InferenceResponse,
|
||||
&json!({
|
||||
"response_id": null,
|
||||
"token_usage": null,
|
||||
"output_items": [{
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "late partial"}]
|
||||
}]
|
||||
}),
|
||||
)?;
|
||||
writer.append(RawTraceEventPayload::InferenceCancelled {
|
||||
inference_call_id: "inference-1".to_string(),
|
||||
reason: "stream mapper noticed cancellation after turn end".to_string(),
|
||||
partial_response_payload: Some(partial_response.clone()),
|
||||
})?;
|
||||
|
||||
let rollout = replay_bundle(temp.path())?;
|
||||
let inference = &rollout.inference_calls["inference-1"];
|
||||
assert_eq!(inference.execution.status, ExecutionStatus::Failed);
|
||||
assert_eq!(inference.execution.ended_seq, Some(turn_end.seq));
|
||||
assert_eq!(
|
||||
inference.raw_response_payload_id,
|
||||
Some(partial_response.raw_payload_id),
|
||||
);
|
||||
assert_eq!(inference.response_item_ids.len(), 1);
|
||||
let response_item_id = &inference.response_item_ids[0];
|
||||
assert_eq!(
|
||||
rollout.conversation_items[response_item_id].body.parts,
|
||||
vec![crate::model::ConversationPart::Text {
|
||||
text: "late partial".to_string(),
|
||||
}],
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -254,6 +254,20 @@ impl TraceReducer {
|
||||
partial_response_payload,
|
||||
)?;
|
||||
}
|
||||
RawTraceEventPayload::InferenceCancelled {
|
||||
inference_call_id,
|
||||
partial_response_payload,
|
||||
..
|
||||
} => {
|
||||
self.complete_inference_call(
|
||||
event.seq,
|
||||
event.wall_time_unix_ms,
|
||||
inference_call_id,
|
||||
ExecutionStatus::Cancelled,
|
||||
/*response_id*/ None,
|
||||
partial_response_payload,
|
||||
)?;
|
||||
}
|
||||
RawTraceEventPayload::ProtocolEventObserved { .. } => {
|
||||
// Protocol wrappers are raw debug breadcrumbs. Typed hooks own
|
||||
// the reduced graph, so these payload refs are retained without
|
||||
|
||||
@@ -187,6 +187,12 @@ impl TraceReducer {
|
||||
&codex_turn_id,
|
||||
&status,
|
||||
)?;
|
||||
self.close_running_inference_calls_for_turn_end(
|
||||
seq,
|
||||
wall_time_unix_ms,
|
||||
&codex_turn_id,
|
||||
&status,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user