Add structured telemetry for response retries (#38452)

## What changed

- Emit trace-safe `codex.retry` events immediately before retry delays, including the attempt number, selected delay, retry layer, and operation.
- Cover HTTP requests, sampling streams, remote compaction, and sampling connection recovery. Track connection-recovery attempts separately from the stream retry budget.
- Add integration coverage for retry timing and terminal behavior across HTTP, SSE, WebSocket, and remote-compaction paths, including rate limits, overloads, connection failures, and `Retry-After` inputs.

GitOrigin-RevId: dae38900a0579cf8ba062c3f3d90ee61851c1c99
This commit is contained in:
Adam Perry @ OpenAI authored and copyberry committed 2026-08-13 23:13:37 +00:00
1 parent 813dc5f08d
commit 1b4ea8b3be
8 files changed
+1779 -2

No files matched your search

+34 -2
View File
@@ -3,7 +3,6 @@ use codex_http_client::TransportError;
use rand::Rng;
use std::future::Future;
use std::time::Duration;
use tokio::time::sleep;
#[derive(Debug, Clone)]
pub struct RetryPolicy {
@@ -48,6 +47,36 @@ pub fn backoff(base: Duration, attempt: u64) -> Duration {
Duration::from_millis((raw as f64 * jitter) as u64)
}
/// Identifies a retry path and its associated trace-event layer.
#[derive(Debug, Clone, Copy)]
pub enum RetryOperation {
HttpRequest,
Sampling,
RemoteCompactionV2,
}
/// Emits retry telemetry at the caller's source location without adding it to normal OTEL logs.
#[macro_export]
macro_rules! record_retry {
($attempt:expr, $delay:expr, $operation:expr $(,)?) => {{
let (layer, operation) = match $operation {
$crate::RetryOperation::HttpRequest => ("http", "request"),
$crate::RetryOperation::Sampling => ("stream", "sampling"),
$crate::RetryOperation::RemoteCompactionV2 => ("stream", "remote_compaction_v2"),
};
::tracing::event!(
target: "codex_otel.trace_safe",
::tracing::Level::TRACE,
event.name = "codex.retry",
retry.attempt = $attempt,
retry.delay_ms = ($delay).as_millis() as u64,
retry.layer = layer,
retry.operation = operation,
);
}};
}
pub async fn run_with_retry<T, F, Fut>(
policy: RetryPolicy,
mut make_req: impl FnMut() -> Request,
@@ -66,7 +95,10 @@ where
.retry_on
.should_retry(&err, attempt, policy.max_attempts) =>
{
sleep(backoff(policy.base_delay, attempt + 1)).await;
let retry_attempt = attempt + 1;
let delay = backoff(policy.base_delay, retry_attempt);
crate::record_retry!(retry_attempt, delay, RetryOperation::HttpRequest);
tokio::time::sleep(delay).await;
}
Err(err) => return Err(err),
}