mirror of
https://github.com/openai/codex.git
synced 2026-11-06 00:30:29 +00:00
Classify streaming rate-limit errors (#40931)
## What changed - Classify `response.failed` events with the `rate_limit_exceeded` code as a distinct retryable error while preserving any parsed retry delay. - Expose the error as `rateLimitExceeded` through the core protocol and app-server schemas after stream retries are exhausted. - Preserve the upstream message for TUI display while keeping it out of telemetry summaries. ## Testing - Cover SSE classification, retry metadata, protocol conversion and serialization, exhausted stream retries, telemetry redaction, and TUI rendering. GitOrigin-RevId: 02dab4d3477dcd7653a58c49c4bd38687a616579
This commit is contained in:
1 parent
7625bd5665
commit
e0c727de04
38 files changed
+198
-26
No files matched your search
@@ -617,6 +617,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
+1
@@ -8013,6 +8013,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
+1
@@ -3974,6 +3974,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
@@ -9,6 +9,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
@@ -38,6 +38,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
@@ -119,6 +119,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
@@ -41,6 +41,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
+1
@@ -41,6 +41,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
@@ -41,6 +41,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
@@ -119,6 +119,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
@@ -41,6 +41,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
@@ -41,6 +41,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
@@ -119,6 +119,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
@@ -41,6 +41,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
@@ -38,6 +38,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
@@ -41,6 +41,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
@@ -38,6 +38,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
@@ -38,6 +38,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
@@ -38,6 +38,7 @@
|
|||||||
"contextWindowExceeded",
|
"contextWindowExceeded",
|
||||||
"sessionBudgetExceeded",
|
"sessionBudgetExceeded",
|
||||||
"usageLimitExceeded",
|
"usageLimitExceeded",
|
||||||
|
"rateLimitExceeded",
|
||||||
"serverOverloaded",
|
"serverOverloaded",
|
||||||
"cyberPolicy",
|
"cyberPolicy",
|
||||||
"misalignmentPolicyViolation",
|
"misalignmentPolicyViolation",
|
||||||
|
|||||||
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -9,4 +9,4 @@ import type { NonSteerableTurnKind } from "./NonSteerableTurnKind";
|
|||||||
* When an upstream HTTP status is available (for example, from the Responses API or a provider),
|
* When an upstream HTTP status is available (for example, from the Responses API or a provider),
|
||||||
* it is forwarded in `httpStatusCode` on the relevant `codexErrorInfo` variant.
|
* it is forwarded in `httpStatusCode` on the relevant `codexErrorInfo` variant.
|
||||||
*/
|
*/
|
||||||
export type CodexErrorInfo = "contextWindowExceeded" | "sessionBudgetExceeded" | "usageLimitExceeded" | "serverOverloaded" | "cyberPolicy" | "misalignmentPolicyViolation" | { "httpConnectionFailed": { httpStatusCode: number | null, } } | { "responseStreamConnectionFailed": { httpStatusCode: number | null, } } | "internalServerError" | "unauthorized" | "badRequest" | "threadRollbackFailed" | "sandboxError" | { "responseStreamDisconnected": { httpStatusCode: number | null, } } | { "responseTooManyFailedAttempts": { httpStatusCode: number | null, } } | { "activeTurnNotSteerable": { turnKind: NonSteerableTurnKind, } } | "other";
|
export type CodexErrorInfo = "contextWindowExceeded" | "sessionBudgetExceeded" | "usageLimitExceeded" | "rateLimitExceeded" | "serverOverloaded" | "cyberPolicy" | "misalignmentPolicyViolation" | { "httpConnectionFailed": { httpStatusCode: number | null, } } | { "responseStreamConnectionFailed": { httpStatusCode: number | null, } } | "internalServerError" | "unauthorized" | "badRequest" | "threadRollbackFailed" | "sandboxError" | { "responseStreamDisconnected": { httpStatusCode: number | null, } } | { "responseTooManyFailedAttempts": { httpStatusCode: number | null, } } | { "activeTurnNotSteerable": { turnKind: NonSteerableTurnKind, } } | "other";
|
||||||
@@ -78,6 +78,7 @@ pub enum CodexErrorInfo {
|
|||||||
ContextWindowExceeded,
|
ContextWindowExceeded,
|
||||||
SessionBudgetExceeded,
|
SessionBudgetExceeded,
|
||||||
UsageLimitExceeded,
|
UsageLimitExceeded,
|
||||||
|
RateLimitExceeded,
|
||||||
ServerOverloaded,
|
ServerOverloaded,
|
||||||
CyberPolicy,
|
CyberPolicy,
|
||||||
MisalignmentPolicyViolation,
|
MisalignmentPolicyViolation,
|
||||||
@@ -125,6 +126,7 @@ impl From<CoreCodexErrorInfo> for CodexErrorInfo {
|
|||||||
CoreCodexErrorInfo::ContextWindowExceeded => CodexErrorInfo::ContextWindowExceeded,
|
CoreCodexErrorInfo::ContextWindowExceeded => CodexErrorInfo::ContextWindowExceeded,
|
||||||
CoreCodexErrorInfo::SessionBudgetExceeded => CodexErrorInfo::SessionBudgetExceeded,
|
CoreCodexErrorInfo::SessionBudgetExceeded => CodexErrorInfo::SessionBudgetExceeded,
|
||||||
CoreCodexErrorInfo::UsageLimitExceeded => CodexErrorInfo::UsageLimitExceeded,
|
CoreCodexErrorInfo::UsageLimitExceeded => CodexErrorInfo::UsageLimitExceeded,
|
||||||
|
CoreCodexErrorInfo::RateLimitExceeded => CodexErrorInfo::RateLimitExceeded,
|
||||||
CoreCodexErrorInfo::ServerOverloaded => CodexErrorInfo::ServerOverloaded,
|
CoreCodexErrorInfo::ServerOverloaded => CodexErrorInfo::ServerOverloaded,
|
||||||
CoreCodexErrorInfo::CyberPolicy => CodexErrorInfo::CyberPolicy,
|
CoreCodexErrorInfo::CyberPolicy => CodexErrorInfo::CyberPolicy,
|
||||||
CoreCodexErrorInfo::MisalignmentPolicyViolation => {
|
CoreCodexErrorInfo::MisalignmentPolicyViolation => {
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ use codex_protocol::permissions::FileSystemSandboxEntry as CoreFileSystemSandbox
|
|||||||
use codex_protocol::permissions::FileSystemSpecialPath as CoreFileSystemSpecialPath;
|
use codex_protocol::permissions::FileSystemSpecialPath as CoreFileSystemSpecialPath;
|
||||||
use codex_protocol::protocol::AgentStatus as CoreAgentStatus;
|
use codex_protocol::protocol::AgentStatus as CoreAgentStatus;
|
||||||
use codex_protocol::protocol::AskForApproval as CoreAskForApproval;
|
use codex_protocol::protocol::AskForApproval as CoreAskForApproval;
|
||||||
|
use codex_protocol::protocol::CodexErrorInfo as CoreCodexErrorInfo;
|
||||||
use codex_protocol::protocol::ConversationTextRole;
|
use codex_protocol::protocol::ConversationTextRole;
|
||||||
use codex_protocol::protocol::ExecCommandSource as CoreExecCommandSource;
|
use codex_protocol::protocol::ExecCommandSource as CoreExecCommandSource;
|
||||||
use codex_protocol::protocol::GranularApprovalConfig as CoreGranularApprovalConfig;
|
use codex_protocol::protocol::GranularApprovalConfig as CoreGranularApprovalConfig;
|
||||||
@@ -4488,11 +4489,19 @@ fn codex_error_info_serializes_http_status_code_in_camel_case() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn codex_error_info_serializes_cyber_policy_in_camel_case() {
|
fn core_error_info_converts_to_camel_case() {
|
||||||
assert_eq!(
|
for (core, expected) in [
|
||||||
serde_json::to_value(CodexErrorInfo::CyberPolicy).unwrap(),
|
(CoreCodexErrorInfo::CyberPolicy, json!("cyberPolicy")),
|
||||||
json!("cyberPolicy")
|
(
|
||||||
);
|
CoreCodexErrorInfo::RateLimitExceeded,
|
||||||
|
json!("rateLimitExceeded"),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(CodexErrorInfo::from(core)).unwrap(),
|
||||||
|
expected
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -1767,6 +1767,7 @@ Ownership rejections for parent-owned Multi-Agent V2 subagents return JSON-RPC e
|
|||||||
- `ContextWindowExceeded`
|
- `ContextWindowExceeded`
|
||||||
- `SessionBudgetExceeded`
|
- `SessionBudgetExceeded`
|
||||||
- `UsageLimitExceeded`
|
- `UsageLimitExceeded`
|
||||||
|
- `rateLimitExceeded`: an upstream rate limit received inside a streaming response; the turn fails with this category only after its existing stream retry budget is exhausted
|
||||||
- `misalignmentPolicyViolation`: a non-retryable request blocked by the misalignment policy
|
- `misalignmentPolicyViolation`: a non-retryable request blocked by the misalignment policy
|
||||||
- `HttpConnectionFailed { httpStatusCode? }`: upstream HTTP failures including 4xx/5xx
|
- `HttpConnectionFailed { httpStatusCode? }`: upstream HTTP failures including 4xx/5xx
|
||||||
- `ResponseStreamConnectionFailed { httpStatusCode? }`: failure to connect to the response SSE stream
|
- `ResponseStreamConnectionFailed { httpStatusCode? }`: failure to connect to the response SSE stream
|
||||||
|
|||||||
@@ -2505,6 +2505,7 @@ fn websocket_error_detail(err: &ApiError) -> String {
|
|||||||
| ApiError::QuotaExceeded
|
| ApiError::QuotaExceeded
|
||||||
| ApiError::UsageNotIncluded
|
| ApiError::UsageNotIncluded
|
||||||
| ApiError::Retryable { .. }
|
| ApiError::Retryable { .. }
|
||||||
|
| ApiError::RateLimitExceeded { .. }
|
||||||
| ApiError::RateLimit(_)
|
| ApiError::RateLimit(_)
|
||||||
| ApiError::InvalidRequest { .. }
|
| ApiError::InvalidRequest { .. }
|
||||||
| ApiError::CyberPolicy { .. }
|
| ApiError::CyberPolicy { .. }
|
||||||
|
|||||||
@@ -29,6 +29,13 @@ pub fn map_api_error(err: ApiError) -> CodexErr {
|
|||||||
None => error,
|
None => error,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
ApiError::RateLimitExceeded { message, delay } => {
|
||||||
|
let error = CodexErr::new(CodexErrorDetails::RateLimitExceeded(message));
|
||||||
|
match delay {
|
||||||
|
Some(delay) => error.with_retry_delay(delay),
|
||||||
|
None => error,
|
||||||
|
}
|
||||||
|
}
|
||||||
ApiError::Stream(msg) => CodexErr::Stream(msg),
|
ApiError::Stream(msg) => CodexErr::Stream(msg),
|
||||||
ApiError::ServerOverloaded => CodexErr::ServerOverloaded,
|
ApiError::ServerOverloaded => CodexErr::ServerOverloaded,
|
||||||
ApiError::Api { status, message } => {
|
ApiError::Api { status, message } => {
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
use base64::Engine;
|
use base64::Engine;
|
||||||
|
use codex_protocol::protocol::CodexErrorInfo;
|
||||||
use codex_protocol::protocol::RateLimitReachedType;
|
use codex_protocol::protocol::RateLimitReachedType;
|
||||||
use pretty_assertions::assert_eq;
|
use pretty_assertions::assert_eq;
|
||||||
|
|
||||||
@@ -12,16 +13,42 @@ fn map_api_error_maps_server_overloaded() {
|
|||||||
#[test]
|
#[test]
|
||||||
fn map_api_error_preserves_retry_delay() {
|
fn map_api_error_preserves_retry_delay() {
|
||||||
let retry_delay = std::time::Duration::from_secs(17);
|
let retry_delay = std::time::Duration::from_secs(17);
|
||||||
let err = map_api_error(ApiError::Retryable {
|
for (error, expected_code, expected_message) in [
|
||||||
message: "retry later".to_string(),
|
(
|
||||||
delay: Some(retry_delay),
|
ApiError::Retryable {
|
||||||
});
|
message: "retry later".to_string(),
|
||||||
|
delay: Some(retry_delay),
|
||||||
assert!(matches!(
|
},
|
||||||
err.details(),
|
CodexErrorInfo::Other,
|
||||||
CodexErrorDetails::Stream(message) if message == "retry later"
|
"stream disconnected before completion: retry later",
|
||||||
));
|
),
|
||||||
assert_eq!(err.retry_delay(), Some(retry_delay));
|
(
|
||||||
|
ApiError::RateLimitExceeded {
|
||||||
|
message: "retry later".to_string(),
|
||||||
|
delay: Some(retry_delay),
|
||||||
|
},
|
||||||
|
CodexErrorInfo::RateLimitExceeded,
|
||||||
|
"rate limit exceeded: retry later",
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let err = map_api_error(error);
|
||||||
|
assert_eq!(
|
||||||
|
(
|
||||||
|
err.to_codex_protocol_error(),
|
||||||
|
err.retry_delay(),
|
||||||
|
err.is_retryable(),
|
||||||
|
err.http_status_code_value(),
|
||||||
|
err.to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
expected_code,
|
||||||
|
Some(retry_delay),
|
||||||
|
true,
|
||||||
|
None,
|
||||||
|
expected_message.to_string(),
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -23,6 +23,11 @@ pub enum ApiError {
|
|||||||
message: String,
|
message: String,
|
||||||
delay: Option<Duration>,
|
delay: Option<Duration>,
|
||||||
},
|
},
|
||||||
|
#[error("rate limit exceeded: {message}")]
|
||||||
|
RateLimitExceeded {
|
||||||
|
message: String,
|
||||||
|
delay: Option<Duration>,
|
||||||
|
},
|
||||||
#[error("rate limit: {0}")]
|
#[error("rate limit: {0}")]
|
||||||
RateLimit(String),
|
RateLimit(String),
|
||||||
#[error("invalid request: {message}")]
|
#[error("invalid request: {message}")]
|
||||||
|
|||||||
@@ -440,7 +440,12 @@ pub fn process_responses_event(
|
|||||||
} else {
|
} else {
|
||||||
let delay = try_parse_retry_after(&error);
|
let delay = try_parse_retry_after(&error);
|
||||||
let message = error.message.unwrap_or_default();
|
let message = error.message.unwrap_or_default();
|
||||||
response_error = ApiError::Retryable { message, delay };
|
response_error = match error.code.as_deref() {
|
||||||
|
Some("rate_limit_exceeded") => {
|
||||||
|
ApiError::RateLimitExceeded { message, delay }
|
||||||
|
}
|
||||||
|
_ => ApiError::Retryable { message, delay },
|
||||||
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return Err(ResponsesEventError::Api(response_error));
|
return Err(ResponsesEventError::Api(response_error));
|
||||||
@@ -1058,7 +1063,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn error_when_error_event() {
|
async fn rate_limit_error_preserves_retry_delay() {
|
||||||
let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_689bcf18d7f08194bf3440ba62fe05d803fee0cdac429894","object":"response","created_at":1755041560,"status":"failed","background":false,"error":{"code":"rate_limit_exceeded","message":"Rate limit reached for gpt-5.1 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more."}, "usage":null,"user":null,"metadata":{}}}"#;
|
let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_689bcf18d7f08194bf3440ba62fe05d803fee0cdac429894","object":"response","created_at":1755041560,"status":"failed","background":false,"error":{"code":"rate_limit_exceeded","message":"Rate limit reached for gpt-5.1 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more."}, "usage":null,"user":null,"metadata":{}}}"#;
|
||||||
|
|
||||||
let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n");
|
let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n");
|
||||||
@@ -1068,14 +1073,55 @@ mod tests {
|
|||||||
assert_eq!(events.len(), 1);
|
assert_eq!(events.len(), 1);
|
||||||
|
|
||||||
match &events[0] {
|
match &events[0] {
|
||||||
Err(ApiError::Retryable { message, delay }) => {
|
Err(ApiError::RateLimitExceeded { message, delay }) => {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
message,
|
message,
|
||||||
"Rate limit reached for gpt-5.1 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more."
|
"Rate limit reached for gpt-5.1 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more."
|
||||||
);
|
);
|
||||||
assert_eq!(*delay, Some(Duration::from_secs_f64(11.054)));
|
assert_eq!(*delay, Some(Duration::from_secs_f64(11.054)));
|
||||||
}
|
}
|
||||||
other => panic!("unexpected second event: {other:?}"),
|
other => panic!("unexpected rate-limit event: {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn failed_response_classification_uses_error_code() {
|
||||||
|
for (code, message) in [
|
||||||
|
("rate_limit_exceeded", "Temporary limit."),
|
||||||
|
(
|
||||||
|
"unknown_error",
|
||||||
|
"Rate limit reached. Please try again in 1s.",
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let event = json!({
|
||||||
|
"type": "response.failed",
|
||||||
|
"response": { "error": { "code": code, "message": message } },
|
||||||
|
});
|
||||||
|
let sse = format!("event: response.failed\ndata: {event}\n\n");
|
||||||
|
let events = collect_events(&[sse.as_bytes()]).await;
|
||||||
|
match (code, events.as_slice()) {
|
||||||
|
(
|
||||||
|
"rate_limit_exceeded",
|
||||||
|
[
|
||||||
|
Err(ApiError::RateLimitExceeded {
|
||||||
|
message: actual,
|
||||||
|
delay,
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
| (
|
||||||
|
"unknown_error",
|
||||||
|
[
|
||||||
|
Err(ApiError::Retryable {
|
||||||
|
message: actual,
|
||||||
|
delay,
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
) => {
|
||||||
|
assert_eq!((actual.as_str(), *delay), (message, None));
|
||||||
|
}
|
||||||
|
_ => panic!("unexpected events for {code}: {events:?}"),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -587,14 +587,17 @@ async fn compact_v2_stream_failure_without_retry_after_exhausts_stream_retries()
|
|||||||
match wait_for_event(&test.codex, |_| true).await {
|
match wait_for_event(&test.codex, |_| true).await {
|
||||||
EventMsg::Error(error) => {
|
EventMsg::Error(error) => {
|
||||||
error_events += 1;
|
error_events += 1;
|
||||||
assert_eq!(error.codex_error_info, Some(CodexErrorInfo::Other));
|
assert_eq!(
|
||||||
|
error.codex_error_info,
|
||||||
|
Some(CodexErrorInfo::RateLimitExceeded)
|
||||||
|
);
|
||||||
assert!(error.message.contains("Rate limit exceeded."));
|
assert!(error.message.contains("Rate limit exceeded."));
|
||||||
}
|
}
|
||||||
EventMsg::StreamError(_) => stream_error_events += 1,
|
EventMsg::StreamError(_) => stream_error_events += 1,
|
||||||
EventMsg::TurnComplete(event) => {
|
EventMsg::TurnComplete(event) => {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
event.error.and_then(|error| error.codex_error_info),
|
event.error.and_then(|error| error.codex_error_info),
|
||||||
Some(CodexErrorInfo::Other)
|
Some(CodexErrorInfo::RateLimitExceeded)
|
||||||
);
|
);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -1001,14 +1004,17 @@ async fn sse_failure_without_retry_after_exhausts_stream_retries() -> Result<()>
|
|||||||
match wait_for_event(&test.codex, |_| true).await {
|
match wait_for_event(&test.codex, |_| true).await {
|
||||||
EventMsg::Error(error) => {
|
EventMsg::Error(error) => {
|
||||||
error_events += 1;
|
error_events += 1;
|
||||||
assert_eq!(error.codex_error_info, Some(CodexErrorInfo::Other));
|
assert_eq!(
|
||||||
|
error.codex_error_info,
|
||||||
|
Some(CodexErrorInfo::RateLimitExceeded)
|
||||||
|
);
|
||||||
assert!(error.message.contains("Rate limit exceeded."));
|
assert!(error.message.contains("Rate limit exceeded."));
|
||||||
}
|
}
|
||||||
EventMsg::StreamError(_) => stream_error_events += 1,
|
EventMsg::StreamError(_) => stream_error_events += 1,
|
||||||
EventMsg::TurnComplete(event) => {
|
EventMsg::TurnComplete(event) => {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
event.error.and_then(|error| error.codex_error_info),
|
event.error.and_then(|error| error.codex_error_info),
|
||||||
Some(CodexErrorInfo::Other)
|
Some(CodexErrorInfo::RateLimitExceeded)
|
||||||
);
|
);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -359,7 +359,10 @@ impl LunaSampler {
|
|||||||
let retryable = match error {
|
let retryable = match error {
|
||||||
LunaSamplerError::ConnectionTimeout
|
LunaSamplerError::ConnectionTimeout
|
||||||
| LunaSamplerError::Api(
|
| LunaSamplerError::Api(
|
||||||
ApiError::Retryable { .. } | ApiError::Stream(_) | ApiError::ServerOverloaded,
|
ApiError::Retryable { .. }
|
||||||
|
| ApiError::RateLimitExceeded { .. }
|
||||||
|
| ApiError::Stream(_)
|
||||||
|
| ApiError::ServerOverloaded,
|
||||||
)
|
)
|
||||||
| LunaSamplerError::Api(ApiError::Transport(
|
| LunaSamplerError::Api(ApiError::Transport(
|
||||||
TransportError::RetryLimit
|
TransportError::RetryLimit
|
||||||
|
|||||||
@@ -91,6 +91,9 @@ pub enum CodexErrorDetails {
|
|||||||
/// The Session loop treats this as a transient error and will automatically retry the turn.
|
/// The Session loop treats this as a transient error and will automatically retry the turn.
|
||||||
#[error("stream disconnected before completion: {0}")]
|
#[error("stream disconnected before completion: {0}")]
|
||||||
Stream(String),
|
Stream(String),
|
||||||
|
/// A retryable upstream rate limit received inside the response stream.
|
||||||
|
#[error("rate limit exceeded: {0}")]
|
||||||
|
RateLimitExceeded(String),
|
||||||
#[error(
|
#[error(
|
||||||
"Codex ran out of room in the model's context window. Start a new thread or clear earlier history before retrying."
|
"Codex ran out of room in the model's context window. Start a new thread or clear earlier history before retrying."
|
||||||
)]
|
)]
|
||||||
@@ -388,6 +391,7 @@ impl CodexErr {
|
|||||||
| CodexErrorDetails::CyberPolicy { .. }
|
| CodexErrorDetails::CyberPolicy { .. }
|
||||||
| CodexErrorDetails::MisalignmentPolicyViolation { .. } => false,
|
| CodexErrorDetails::MisalignmentPolicyViolation { .. } => false,
|
||||||
CodexErrorDetails::Stream(..)
|
CodexErrorDetails::Stream(..)
|
||||||
|
| CodexErrorDetails::RateLimitExceeded(_)
|
||||||
| CodexErrorDetails::Timeout
|
| CodexErrorDetails::Timeout
|
||||||
| CodexErrorDetails::RequestTimeout
|
| CodexErrorDetails::RequestTimeout
|
||||||
| CodexErrorDetails::UnexpectedStatus(_)
|
| CodexErrorDetails::UnexpectedStatus(_)
|
||||||
@@ -424,6 +428,7 @@ impl CodexErr {
|
|||||||
match &self.details {
|
match &self.details {
|
||||||
CodexErrorDetails::ContextWindowExceeded => CodexErrorInfo::ContextWindowExceeded,
|
CodexErrorDetails::ContextWindowExceeded => CodexErrorInfo::ContextWindowExceeded,
|
||||||
CodexErrorDetails::SessionBudgetExceeded => CodexErrorInfo::SessionBudgetExceeded,
|
CodexErrorDetails::SessionBudgetExceeded => CodexErrorInfo::SessionBudgetExceeded,
|
||||||
|
CodexErrorDetails::RateLimitExceeded(_) => CodexErrorInfo::RateLimitExceeded,
|
||||||
CodexErrorDetails::UsageLimitReached(_)
|
CodexErrorDetails::UsageLimitReached(_)
|
||||||
| CodexErrorDetails::QuotaExceeded
|
| CodexErrorDetails::QuotaExceeded
|
||||||
| CodexErrorDetails::UsageNotIncluded => CodexErrorInfo::UsageLimitExceeded,
|
| CodexErrorDetails::UsageNotIncluded => CodexErrorInfo::UsageLimitExceeded,
|
||||||
|
|||||||
@@ -36,6 +36,10 @@ fn codex_err_debug_preserves_legacy_shape() {
|
|||||||
fn retryability_preserves_error_details_distinctions() {
|
fn retryability_preserves_error_details_distinctions() {
|
||||||
let errors = [
|
let errors = [
|
||||||
(CodexErr::ServerOverloaded, false),
|
(CodexErr::ServerOverloaded, false),
|
||||||
|
(
|
||||||
|
CodexErr::new(CodexErrorDetails::RateLimitExceeded("retry later".into())),
|
||||||
|
true,
|
||||||
|
),
|
||||||
(
|
(
|
||||||
CodexErr::RetryLimit(RetryLimitReachedError {
|
CodexErr::RetryLimit(RetryLimitReachedError {
|
||||||
status: StatusCode::TOO_MANY_REQUESTS,
|
status: StatusCode::TOO_MANY_REQUESTS,
|
||||||
|
|||||||
@@ -1820,6 +1820,7 @@ pub enum CodexErrorInfo {
|
|||||||
ContextWindowExceeded,
|
ContextWindowExceeded,
|
||||||
SessionBudgetExceeded,
|
SessionBudgetExceeded,
|
||||||
UsageLimitExceeded,
|
UsageLimitExceeded,
|
||||||
|
RateLimitExceeded,
|
||||||
ServerOverloaded,
|
ServerOverloaded,
|
||||||
CyberPolicy,
|
CyberPolicy,
|
||||||
MisalignmentPolicyViolation,
|
MisalignmentPolicyViolation,
|
||||||
@@ -1859,6 +1860,7 @@ impl CodexErrorInfo {
|
|||||||
Self::ContextWindowExceeded
|
Self::ContextWindowExceeded
|
||||||
| Self::SessionBudgetExceeded
|
| Self::SessionBudgetExceeded
|
||||||
| Self::UsageLimitExceeded
|
| Self::UsageLimitExceeded
|
||||||
|
| Self::RateLimitExceeded
|
||||||
| Self::ServerOverloaded
|
| Self::ServerOverloaded
|
||||||
| Self::CyberPolicy
|
| Self::CyberPolicy
|
||||||
| Self::MisalignmentPolicyViolation
|
| Self::MisalignmentPolicyViolation
|
||||||
|
|||||||
@@ -80,6 +80,7 @@ pub fn telemetry_api_error_message(error: &ApiError) -> String {
|
|||||||
ApiError::QuotaExceeded => "quota exceeded".to_string(),
|
ApiError::QuotaExceeded => "quota exceeded".to_string(),
|
||||||
ApiError::UsageNotIncluded => "usage not included".to_string(),
|
ApiError::UsageNotIncluded => "usage not included".to_string(),
|
||||||
ApiError::Retryable { .. } => "retryable error".to_string(),
|
ApiError::Retryable { .. } => "retryable error".to_string(),
|
||||||
|
ApiError::RateLimitExceeded { .. } => "rate limit exceeded".to_string(),
|
||||||
ApiError::RateLimit(_) => "rate limit".to_string(),
|
ApiError::RateLimit(_) => "rate limit".to_string(),
|
||||||
ApiError::InvalidRequest { .. } => "invalid request".to_string(),
|
ApiError::InvalidRequest { .. } => "invalid request".to_string(),
|
||||||
ApiError::CyberPolicy { .. } => "cyber policy".to_string(),
|
ApiError::CyberPolicy { .. } => "cyber policy".to_string(),
|
||||||
@@ -134,7 +135,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn telemetry_error_messages_omit_http_bodies() {
|
fn telemetry_error_messages_omit_upstream_bodies() {
|
||||||
let transport = TransportError::Http {
|
let transport = TransportError::Http {
|
||||||
status: StatusCode::UNAUTHORIZED,
|
status: StatusCode::UNAUTHORIZED,
|
||||||
url: Some("https://chatgpt.com/backend-api/codex/responses".to_string()),
|
url: Some("https://chatgpt.com/backend-api/codex/responses".to_string()),
|
||||||
@@ -147,6 +148,13 @@ mod tests {
|
|||||||
telemetry_api_error_message(&ApiError::Transport(transport)),
|
telemetry_api_error_message(&ApiError::Transport(transport)),
|
||||||
"http 401"
|
"http 401"
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
telemetry_api_error_message(&ApiError::RateLimitExceeded {
|
||||||
|
message: "private upstream diagnostic".to_string(),
|
||||||
|
delay: Some(std::time::Duration::from_secs(1)),
|
||||||
|
}),
|
||||||
|
"rate limit exceeded"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
+5
@@ -0,0 +1,5 @@
|
|||||||
|
---
|
||||||
|
source: tui/src/chatwidget/tests/app_server.rs
|
||||||
|
expression: lines_to_single_string(&lines)
|
||||||
|
---
|
||||||
|
■ rate limit exceeded: Please try again in 10s.
|
||||||
@@ -1,4 +1,6 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
use codex_protocol::error::CodexErr;
|
||||||
|
use codex_protocol::error::CodexErrorDetails;
|
||||||
use pretty_assertions::assert_eq;
|
use pretty_assertions::assert_eq;
|
||||||
|
|
||||||
const SAFETY_BUFFERING_HEADER_TEXT: &str =
|
const SAFETY_BUFFERING_HEADER_TEXT: &str =
|
||||||
@@ -1361,6 +1363,26 @@ async fn live_app_server_stream_recovery_restores_previous_status_header() {
|
|||||||
assert!(chat.status_state.retry_status_header.is_none());
|
assert!(chat.status_state.retry_status_header.is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn live_app_server_rate_limit_error_renders_upstream_message() {
|
||||||
|
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
|
||||||
|
let error = CodexErr::from(CodexErrorDetails::RateLimitExceeded(
|
||||||
|
"Please try again in 10s.".to_string(),
|
||||||
|
));
|
||||||
|
|
||||||
|
handle_error(
|
||||||
|
&mut chat,
|
||||||
|
error.to_string(),
|
||||||
|
Some(error.to_codex_protocol_error().into()),
|
||||||
|
);
|
||||||
|
|
||||||
|
let lines = drain_insert_history(&mut rx)
|
||||||
|
.into_iter()
|
||||||
|
.flatten()
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
assert_chatwidget_snapshot!("rate_limit_exceeded_error", lines_to_single_string(&lines));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn live_app_server_server_overloaded_error_renders_warning() {
|
async fn live_app_server_server_overloaded_error_renders_warning() {
|
||||||
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
|
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
|
||||||
|
|||||||
Reference in new issue
Block a user