mirror of
https://github.com/openai/codex.git
synced 2026-11-18 07:00:27 +00:00
Add structured telemetry for response retries (#38452)
## What changed - Emit trace-safe `codex.retry` events immediately before retry delays, including the attempt number, selected delay, retry layer, and operation. - Cover HTTP requests, sampling streams, remote compaction, and sampling connection recovery. Track connection-recovery attempts separately from the stream retry budget. - Add integration coverage for retry timing and terminal behavior across HTTP, SSE, WebSocket, and remote-compaction paths, including rate limits, overloads, connection failures, and `Retry-After` inputs. GitOrigin-RevId: dae38900a0579cf8ba062c3f3d90ee61851c1c99
This commit is contained in:
1 parent
813dc5f08d
commit
1b4ea8b3be
8 files changed
+1779
-2
No files matched your search
@@ -3,7 +3,6 @@ use codex_http_client::TransportError;
|
||||
use rand::Rng;
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
use tokio::time::sleep;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RetryPolicy {
|
||||
@@ -48,6 +47,36 @@ pub fn backoff(base: Duration, attempt: u64) -> Duration {
|
||||
Duration::from_millis((raw as f64 * jitter) as u64)
|
||||
}
|
||||
|
||||
/// Identifies a retry path and its associated trace-event layer.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum RetryOperation {
|
||||
HttpRequest,
|
||||
Sampling,
|
||||
RemoteCompactionV2,
|
||||
}
|
||||
|
||||
/// Emits retry telemetry at the caller's source location without adding it to normal OTEL logs.
|
||||
#[macro_export]
|
||||
macro_rules! record_retry {
|
||||
($attempt:expr, $delay:expr, $operation:expr $(,)?) => {{
|
||||
let (layer, operation) = match $operation {
|
||||
$crate::RetryOperation::HttpRequest => ("http", "request"),
|
||||
$crate::RetryOperation::Sampling => ("stream", "sampling"),
|
||||
$crate::RetryOperation::RemoteCompactionV2 => ("stream", "remote_compaction_v2"),
|
||||
};
|
||||
|
||||
::tracing::event!(
|
||||
target: "codex_otel.trace_safe",
|
||||
::tracing::Level::TRACE,
|
||||
event.name = "codex.retry",
|
||||
retry.attempt = $attempt,
|
||||
retry.delay_ms = ($delay).as_millis() as u64,
|
||||
retry.layer = layer,
|
||||
retry.operation = operation,
|
||||
);
|
||||
}};
|
||||
}
|
||||
|
||||
pub async fn run_with_retry<T, F, Fut>(
|
||||
policy: RetryPolicy,
|
||||
mut make_req: impl FnMut() -> Request,
|
||||
@@ -66,7 +95,10 @@ where
|
||||
.retry_on
|
||||
.should_retry(&err, attempt, policy.max_attempts) =>
|
||||
{
|
||||
sleep(backoff(policy.base_delay, attempt + 1)).await;
|
||||
let retry_attempt = attempt + 1;
|
||||
let delay = backoff(policy.base_delay, retry_attempt);
|
||||
crate::record_retry!(retry_attempt, delay, RetryOperation::HttpRequest);
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user