mirror of
https://github.com/openai/codex.git
synced 2026-11-14 14:49:31 +00:00
Fix retry classification for throttling and quota errors (#45602)
## Why `slow_down` errors were treated as terminal server overloads, while exhausted credit balances and spend limits fell through to retryable stream errors. ## What changed - Classify `slow_down` as a retryable rate limit in HTTP 503 responses and SSE failures, preserving the server message and parsing retry delays from SSE error messages. - Map `credit_balance_exhausted`, `organization_spend_limit_exceeded`, and `project_spend_limit_exceeded` SSE errors to quota exhaustion so they terminate without retries. ## Testing Extend tests to cover HTTP error classification and retryability, `slow_down` stream retry exhaustion and message-provided delays, and a single `UsageLimitExceeded` error with no retries for each quota error code. GitOrigin-RevId: 7985eba71ba4a6fe897a45957e448fadd42f3963
This commit is contained in:
1 parent
fc269b66ad
commit
31ffe2bc9a
5 files changed
+83
-38
No files matched your search
@@ -1,5 +1,6 @@
|
||||
use anyhow::Result;
|
||||
use codex_core::TurnInputRequest;
|
||||
use codex_protocol::protocol::CodexErrorInfo;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use core_test_support::responses::ev_response_created;
|
||||
@@ -12,8 +13,12 @@ use core_test_support::wait_for_event;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
|
||||
#[test_case::test_case("insufficient_quota"; "quota")]
|
||||
#[test_case::test_case("credit_balance_exhausted"; "credit_balance")]
|
||||
#[test_case::test_case("organization_spend_limit_exceeded"; "organization_spend_limit")]
|
||||
#[test_case::test_case("project_spend_limit_exceeded"; "project_spend_limit")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn quota_exceeded_emits_single_error_event() -> Result<()> {
|
||||
async fn quota_exceeded_emits_single_error_event(code: &str) -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let server = start_mock_server().await;
|
||||
@@ -28,7 +33,7 @@ async fn quota_exceeded_emits_single_error_event() -> Result<()> {
|
||||
"response": {
|
||||
"id": "resp-1",
|
||||
"error": {
|
||||
"code": "insufficient_quota",
|
||||
"code": code,
|
||||
"message": "You exceeded your current quota, please check your plan and billing details."
|
||||
}
|
||||
}
|
||||
@@ -37,15 +42,14 @@ async fn quota_exceeded_emits_single_error_event() -> Result<()> {
|
||||
)
|
||||
.await;
|
||||
|
||||
let test = builder.build(&server).await?;
|
||||
let test = builder.build_with_auto_env(&server).await?;
|
||||
|
||||
test.codex
|
||||
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
|
||||
text: "quota?".into(),
|
||||
text_elements: Vec::new(),
|
||||
}]))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
|
||||
let mut error_events = 0;
|
||||
|
||||
@@ -59,6 +63,10 @@ async fn quota_exceeded_emits_single_error_event() -> Result<()> {
|
||||
err.message,
|
||||
"Quota exceeded. Check your plan and billing details."
|
||||
);
|
||||
assert_eq!(
|
||||
err.codex_error_info,
|
||||
Some(CodexErrorInfo::UsageLimitExceeded)
|
||||
);
|
||||
}
|
||||
EventMsg::TurnComplete(_) => break,
|
||||
_ => {}
|
||||
@@ -66,6 +74,14 @@ async fn quota_exceeded_emits_single_error_event() -> Result<()> {
|
||||
}
|
||||
|
||||
assert_eq!(error_events, 1, "expected exactly one Codex:Error event");
|
||||
let requests = server.received_requests().await.expect("recorded requests");
|
||||
assert_eq!(
|
||||
requests
|
||||
.iter()
|
||||
.filter(|request| request.url.path().ends_with("/responses"))
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -989,8 +989,10 @@ async fn sse_failure_uses_local_backoff_despite_retry_after() -> Result<()> {
|
||||
}
|
||||
|
||||
/// Headerless sampled stream rate limits exhaust retries before one terminal error.
|
||||
#[test_case::test_case("rate_limit_exceeded"; "rate_limit")]
|
||||
#[test_case::test_case("slow_down"; "slow_down")]
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn sse_failure_without_retry_after_exhausts_stream_retries() -> Result<()> {
|
||||
async fn sse_failure_without_retry_after_exhausts_stream_retries(code: &str) -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let mut telemetry = RetryTelemetryCapture::install();
|
||||
@@ -1000,12 +1002,12 @@ async fn sse_failure_without_retry_after_exhausts_stream_retries() -> Result<()>
|
||||
vec![
|
||||
responses::sse_response(responses::sse_failed(
|
||||
"rate-limited",
|
||||
"rate_limit_exceeded",
|
||||
code,
|
||||
"Rate limit exceeded.",
|
||||
)),
|
||||
responses::sse_response(responses::sse_failed(
|
||||
"still-rate-limited",
|
||||
"rate_limit_exceeded",
|
||||
code,
|
||||
"Rate limit exceeded.",
|
||||
)),
|
||||
],
|
||||
@@ -1069,8 +1071,10 @@ async fn sse_failure_without_retry_after_exhausts_stream_retries() -> Result<()>
|
||||
}
|
||||
|
||||
/// Rate-limit messages already provide an exact retry delay without an HTTP header.
|
||||
#[test_case::test_case("rate_limit_exceeded"; "rate_limit")]
|
||||
#[test_case::test_case("slow_down"; "slow_down")]
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn sse_rate_limit_message_uses_server_advised_retry_delay() -> Result<()> {
|
||||
async fn sse_rate_limit_message_uses_server_advised_retry_delay(code: &str) -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let mut telemetry = RetryTelemetryCapture::install();
|
||||
@@ -1080,7 +1084,7 @@ async fn sse_rate_limit_message_uses_server_advised_retry_delay() -> Result<()>
|
||||
vec![
|
||||
responses::sse_response(responses::sse_failed(
|
||||
"rate-limited",
|
||||
"rate_limit_exceeded",
|
||||
code,
|
||||
"Rate limit exceeded. Please try again in 1s.",
|
||||
)),
|
||||
responses::sse_response(responses::sse(vec![
|
||||
|
||||
Reference in new issue
Block a user