mirror of
https://github.com/openai/codex.git
synced 2026-11-17 14:58:36 +00:00
[rollout-trace] Include x-request-id in rollout trace. (#20066)
## Why Rollout traces need an identifier that can be used to correlate a Codex inference with upstream Responses API, proxy, and engine logs. The reduced trace model already exposed `upstream_request_id`, but it was being populated from the Responses API `response.id`. That value is useful for `previous_response_id` chaining, but it is not the transport request id that upstream systems key on. This PR separates those concepts so trace consumers can reliably answer both questions: - which Responses API response did this inference produce? - which upstream request handled it? ## Structure The change keeps the upstream request id at the same lifecycle level as the provider stream: - `codex-api` captures the `x-request-id` HTTP response header when the SSE stream is created and exposes it on `ResponseStream`. Fixture and websocket streams set the field to `None` because they do not have that HTTP response header. - `codex-core` carries that stream-level id into `InferenceTraceAttempt` when recording terminal stream outcomes. Completed, failed, cancelled, dropped-stream, and pre-response error paths all record the id when it is available. - `rollout-trace` now records both identifiers in raw terminal inference events and response payloads: `response_id` for the Responses API `response.id`, and `upstream_request_id` for `x-request-id`. - The reducer stores both fields on `InferenceCall`. It also uses `response_id` for `previous_response_id` conversation linking, which removes the old accidental dependency on the misnamed `upstream_request_id` field. - Terminal inference reduction now consumes the full terminal payload (`InferenceCompleted`, `InferenceFailed`, or `InferenceCancelled`) in one place. That keeps status, partial payloads, response ids, and upstream request ids consistent across success, failure, cancellation, and late stream-mapper events. ## Why This Shape `x-request-id` is a property of the HTTP/provider response envelope, not an SSE event. Capturing it once in `codex-api` and plumbing it through terminal trace recording avoids trying to infer the value from stream contents, and it preserves the id even when the stream fails or is cancelled after only partial output. Keeping `response_id` separate from `upstream_request_id` also makes the reduced trace model less surprising: `response_id` remains the conversation-continuation id, while `upstream_request_id` is the operational correlation id for upstream debugging. ## Validation The PR updates trace and reducer coverage for: - reading `x-request-id` from SSE response headers; - storing the true upstream request id on completed inference calls; - preserving upstream request ids for cancelled and late-cancelled inference streams; - keeping `previous_response_id` reconstruction tied to `response_id` rather than transport request ids.
This commit is contained in:
1 parent
10e2a73b3c
commit
89698ad1c3
17 files changed
+240
-90
No files matched your search
@@ -64,6 +64,7 @@ fn code_cell_lifecycle_links_nested_tools_waits_and_outputs() -> anyhow::Result<
|
||||
writer.append(RawTraceEventPayload::InferenceCompleted {
|
||||
inference_call_id: "inference-1".to_string(),
|
||||
response_id: Some("resp-1".to_string()),
|
||||
upstream_request_id: None,
|
||||
response_payload: response,
|
||||
})?;
|
||||
writer.append_with_context(
|
||||
@@ -247,6 +248,7 @@ fn fast_code_cell_lifecycle_waits_for_source_item() -> anyhow::Result<()> {
|
||||
writer.append(RawTraceEventPayload::InferenceCompleted {
|
||||
inference_call_id: "inference-1".to_string(),
|
||||
response_id: Some("resp-1".to_string()),
|
||||
upstream_request_id: None,
|
||||
response_payload: response,
|
||||
})?;
|
||||
|
||||
@@ -301,6 +303,7 @@ fn cancelled_turn_terminates_unfinished_code_cell() -> anyhow::Result<()> {
|
||||
writer.append(RawTraceEventPayload::InferenceCompleted {
|
||||
inference_call_id: "inference-1".to_string(),
|
||||
response_id: Some("resp-1".to_string()),
|
||||
upstream_request_id: None,
|
||||
response_payload: response,
|
||||
})?;
|
||||
writer.append_with_context(
|
||||
@@ -388,6 +391,7 @@ fn runtime_code_cell_ids_can_repeat_across_threads() -> anyhow::Result<()> {
|
||||
writer.append(RawTraceEventPayload::InferenceCompleted {
|
||||
inference_call_id: inference_call_id.to_string(),
|
||||
response_id: Some(format!("resp-{thread_id}")),
|
||||
upstream_request_id: None,
|
||||
response_payload: response,
|
||||
})?;
|
||||
writer.append_with_context(
|
||||
|
||||
@@ -76,7 +76,7 @@ impl TraceReducer {
|
||||
.values()
|
||||
.find(|inference| {
|
||||
inference.thread_id == thread_id
|
||||
&& inference.upstream_request_id.as_deref() == Some(previous_response_id)
|
||||
&& inference.response_id.as_deref() == Some(previous_response_id)
|
||||
})
|
||||
.map(|inference| {
|
||||
let mut ids = inference.request_item_ids.clone();
|
||||
|
||||
@@ -13,6 +13,7 @@ use crate::model::InferenceCall;
|
||||
use crate::model::InferenceCallId;
|
||||
use crate::payload::RawPayloadRef;
|
||||
use crate::raw_event::RawEventSeq;
|
||||
use crate::raw_event::RawTraceEventPayload;
|
||||
|
||||
/// Raw inference-start fields after dispatch has stripped the common event envelope.
|
||||
///
|
||||
@@ -91,6 +92,7 @@ impl TraceReducer {
|
||||
},
|
||||
model: started.model,
|
||||
provider_name: started.provider_name,
|
||||
response_id: None,
|
||||
upstream_request_id: None,
|
||||
request_item_ids,
|
||||
response_item_ids: Vec::new(),
|
||||
@@ -137,11 +139,49 @@ impl TraceReducer {
|
||||
&mut self,
|
||||
seq: RawEventSeq,
|
||||
wall_time_unix_ms: i64,
|
||||
inference_call_id: InferenceCallId,
|
||||
status: ExecutionStatus,
|
||||
response_id: Option<String>,
|
||||
response_payload: Option<RawPayloadRef>,
|
||||
payload: RawTraceEventPayload,
|
||||
) -> Result<()> {
|
||||
let (inference_call_id, status, response_id, upstream_request_id, response_payload) =
|
||||
match payload {
|
||||
RawTraceEventPayload::InferenceCompleted {
|
||||
inference_call_id,
|
||||
response_id,
|
||||
upstream_request_id,
|
||||
response_payload,
|
||||
} => (
|
||||
inference_call_id,
|
||||
ExecutionStatus::Completed,
|
||||
response_id,
|
||||
upstream_request_id,
|
||||
Some(response_payload),
|
||||
),
|
||||
RawTraceEventPayload::InferenceFailed {
|
||||
inference_call_id,
|
||||
upstream_request_id,
|
||||
partial_response_payload,
|
||||
..
|
||||
} => (
|
||||
inference_call_id,
|
||||
ExecutionStatus::Failed,
|
||||
None,
|
||||
upstream_request_id,
|
||||
partial_response_payload,
|
||||
),
|
||||
RawTraceEventPayload::InferenceCancelled {
|
||||
inference_call_id,
|
||||
upstream_request_id,
|
||||
partial_response_payload,
|
||||
..
|
||||
} => (
|
||||
inference_call_id,
|
||||
ExecutionStatus::Cancelled,
|
||||
None,
|
||||
upstream_request_id,
|
||||
partial_response_payload,
|
||||
),
|
||||
_ => bail!("complete_inference_call received a non-terminal inference event"),
|
||||
};
|
||||
|
||||
if !self
|
||||
.rollout
|
||||
.inference_calls
|
||||
@@ -156,23 +196,31 @@ impl TraceReducer {
|
||||
self.reduce_inference_response(wall_time_unix_ms, &inference_call_id, payload)
|
||||
})
|
||||
.transpose()?;
|
||||
let Some(inference) = self.rollout.inference_calls.get_mut(&inference_call_id) else {
|
||||
bail!("inference call {inference_call_id} disappeared during response reduction");
|
||||
};
|
||||
// Turn-end cleanup can close a stream before the async mapper observes
|
||||
// cancellation. Preserve that terminal status while still retaining any
|
||||
// late partial response evidence from the mapper.
|
||||
if inference.execution.status == ExecutionStatus::Running {
|
||||
inference.execution.ended_at_unix_ms = Some(wall_time_unix_ms);
|
||||
inference.execution.ended_seq = Some(seq);
|
||||
inference.execution.status = status;
|
||||
inference.upstream_request_id = response_id;
|
||||
}
|
||||
if let Some(response_payload) = response_payload {
|
||||
inference.raw_response_payload_id = Some(response_payload.raw_payload_id);
|
||||
}
|
||||
if let Some(response_item_ids) = response_item_ids {
|
||||
inference.response_item_ids = response_item_ids;
|
||||
{
|
||||
let Some(inference) = self.rollout.inference_calls.get_mut(&inference_call_id) else {
|
||||
bail!("inference call {inference_call_id} disappeared during response reduction");
|
||||
};
|
||||
inference.response_id = response_id;
|
||||
// Turn-end cleanup can close a stream before the async mapper observes
|
||||
// cancellation. Preserve that terminal status while still retaining any
|
||||
// late partial response evidence from the mapper.
|
||||
if inference.execution.status == ExecutionStatus::Running {
|
||||
inference.execution.ended_at_unix_ms = Some(wall_time_unix_ms);
|
||||
inference.execution.ended_seq = Some(seq);
|
||||
inference.execution.status = status;
|
||||
}
|
||||
// Turn-end cleanup can mark an inference terminal before the stream
|
||||
// mapper records its late partial payload. Keep the server request
|
||||
// id from that late payload even when the status is already closed.
|
||||
if let Some(upstream_request_id) = upstream_request_id {
|
||||
inference.upstream_request_id = Some(upstream_request_id);
|
||||
}
|
||||
if let Some(response_payload) = response_payload {
|
||||
inference.raw_response_payload_id = Some(response_payload.raw_payload_id);
|
||||
}
|
||||
if let Some(response_item_ids) = response_item_ids {
|
||||
inference.response_item_ids = response_item_ids;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -40,6 +40,7 @@ fn cancelled_inference_reduces_partial_response_items() -> anyhow::Result<()> {
|
||||
)?;
|
||||
writer.append(RawTraceEventPayload::InferenceCancelled {
|
||||
inference_call_id: "inference-1".to_string(),
|
||||
upstream_request_id: Some("req-cancelled".to_string()),
|
||||
reason: "test interruption".to_string(),
|
||||
partial_response_payload: Some(partial_response),
|
||||
})?;
|
||||
@@ -49,6 +50,10 @@ fn cancelled_inference_reduces_partial_response_items() -> anyhow::Result<()> {
|
||||
let response_item_id = &inference.response_item_ids[0];
|
||||
|
||||
assert_eq!(inference.execution.status, ExecutionStatus::Cancelled);
|
||||
assert_eq!(
|
||||
inference.upstream_request_id,
|
||||
Some("req-cancelled".to_string()),
|
||||
);
|
||||
assert_eq!(inference.response_item_ids.len(), 1);
|
||||
assert_eq!(
|
||||
rollout.conversation_items[response_item_id].kind,
|
||||
@@ -123,6 +128,7 @@ fn late_cancelled_inference_preserves_turn_end_status() -> anyhow::Result<()> {
|
||||
)?;
|
||||
writer.append(RawTraceEventPayload::InferenceCancelled {
|
||||
inference_call_id: "inference-1".to_string(),
|
||||
upstream_request_id: Some("req-late-cancelled".to_string()),
|
||||
reason: "stream mapper noticed cancellation after turn end".to_string(),
|
||||
partial_response_payload: Some(partial_response.clone()),
|
||||
})?;
|
||||
@@ -135,6 +141,10 @@ fn late_cancelled_inference_preserves_turn_end_status() -> anyhow::Result<()> {
|
||||
inference.raw_response_payload_id,
|
||||
Some(partial_response.raw_payload_id),
|
||||
);
|
||||
assert_eq!(
|
||||
inference.upstream_request_id,
|
||||
Some("req-late-cancelled".to_string()),
|
||||
);
|
||||
assert_eq!(inference.response_item_ids.len(), 1);
|
||||
let response_item_id = &inference.response_item_ids[0];
|
||||
assert_eq!(
|
||||
|
||||
@@ -226,47 +226,10 @@ impl TraceReducer {
|
||||
},
|
||||
)?;
|
||||
}
|
||||
RawTraceEventPayload::InferenceCompleted {
|
||||
inference_call_id,
|
||||
response_id,
|
||||
response_payload,
|
||||
} => {
|
||||
self.complete_inference_call(
|
||||
event.seq,
|
||||
event.wall_time_unix_ms,
|
||||
inference_call_id,
|
||||
ExecutionStatus::Completed,
|
||||
response_id,
|
||||
Some(response_payload),
|
||||
)?;
|
||||
}
|
||||
RawTraceEventPayload::InferenceFailed {
|
||||
inference_call_id,
|
||||
partial_response_payload,
|
||||
..
|
||||
} => {
|
||||
self.complete_inference_call(
|
||||
event.seq,
|
||||
event.wall_time_unix_ms,
|
||||
inference_call_id,
|
||||
ExecutionStatus::Failed,
|
||||
/*response_id*/ None,
|
||||
partial_response_payload,
|
||||
)?;
|
||||
}
|
||||
RawTraceEventPayload::InferenceCancelled {
|
||||
inference_call_id,
|
||||
partial_response_payload,
|
||||
..
|
||||
} => {
|
||||
self.complete_inference_call(
|
||||
event.seq,
|
||||
event.wall_time_unix_ms,
|
||||
inference_call_id,
|
||||
ExecutionStatus::Cancelled,
|
||||
/*response_id*/ None,
|
||||
partial_response_payload,
|
||||
)?;
|
||||
payload @ (RawTraceEventPayload::InferenceCompleted { .. }
|
||||
| RawTraceEventPayload::InferenceFailed { .. }
|
||||
| RawTraceEventPayload::InferenceCancelled { .. }) => {
|
||||
self.complete_inference_call(event.seq, event.wall_time_unix_ms, payload)?;
|
||||
}
|
||||
RawTraceEventPayload::ProtocolEventObserved { .. } => {
|
||||
// Protocol wrappers are raw debug breadcrumbs. Typed hooks own
|
||||
|
||||
@@ -146,6 +146,7 @@ pub(crate) fn append_inference_completion(
|
||||
writer.append(RawTraceEventPayload::InferenceCompleted {
|
||||
inference_call_id: inference_call_id.to_string(),
|
||||
response_id: Some(response_id.to_string()),
|
||||
upstream_request_id: None,
|
||||
response_payload,
|
||||
})?;
|
||||
Ok(())
|
||||
@@ -184,6 +185,7 @@ pub(crate) fn append_completed_inference(
|
||||
RawTraceEventPayload::InferenceCompleted {
|
||||
inference_call_id: inference_id.to_string(),
|
||||
response_id: Some(format!("resp-{inference_id}")),
|
||||
upstream_request_id: None,
|
||||
response_payload: response,
|
||||
},
|
||||
)?;
|
||||
|
||||
@@ -551,6 +551,7 @@ fn append_inference_with_tool_call(writer: &TraceWriter) -> anyhow::Result<()> {
|
||||
writer.append(RawTraceEventPayload::InferenceCompleted {
|
||||
inference_call_id: "inference-1".to_string(),
|
||||
response_id: Some("resp-1".to_string()),
|
||||
upstream_request_id: None,
|
||||
response_payload: response,
|
||||
})?;
|
||||
Ok(())
|
||||
|
||||
Reference in new issue
Block a user