Retry file blob uploads on HTTP 502 and 504 (#47926)

## Why

File blob uploads previously failed immediately on HTTP 502 and 504 responses, even though these transient gateway errors can recover on retry.

## What changed

Treat HTTP 502 and 504 as retryable alongside 503, using the existing attempt limit, upload deadline, and server-provided retry delays.

## Testing

Expand upload retry tests to cover recovery from 502 and 504, mixed transient responses, exhausted attempts, and retry delays exceeding the remaining budget. Verify that 403, 409, and 500 responses are not retried, including responses without an Azure error code.

GitOrigin-RevId: a8e2f6bcf69338fb6d24f4c2d1ee333fb77bdbc0
This commit is contained in:
mtsui-oai
2026-09-24 19:52:26 +00:00
committed by copyberry
parent 1866e51677
commit 0037b39cc1
2 changed files with 25 additions and 8 deletions
+6 -1
View File
@@ -291,7 +291,12 @@ where
azure_request_id: azure_request_id.clone(),
azure_error_code: azure_error_code.clone(),
},
upload_status == StatusCode::SERVICE_UNAVAILABLE,
matches!(
upload_status,
StatusCode::BAD_GATEWAY
| StatusCode::SERVICE_UNAVAILABLE
| StatusCode::GATEWAY_TIMEOUT
),
"http_status",
)
}
+19 -7
View File
@@ -50,12 +50,21 @@ async fn mock_upload(
}
#[tokio::test]
async fn retries_503_within_budget_but_not_4xx() {
async fn retries_transient_upload_statuses_within_budget() {
for (statuses, azure_code, retry_after) in [
(vec![503, 200], "ServerBusy", None),
(vec![503; 5], "ServerBusy", None),
(vec![403], "AuthenticationFailed", None),
(vec![503], "ServerBusy", Some("301")),
(vec![502, 200], None, None),
(vec![503, 200], Some("ServerBusy"), None),
(vec![504, 200], None, None),
(vec![502, 504, 503, 200], None, None),
(vec![502; 5], None, None),
(vec![503; 5], Some("ServerBusy"), None),
(vec![504; 5], None, None),
(vec![403], Some("AuthenticationFailed"), None),
(vec![409], None, None),
(vec![500], None, None),
(vec![502], None, Some("301")),
(vec![503], Some("ServerBusy"), Some("301")),
(vec![504], None, Some("301")),
] {
let server = MockServer::start().await;
let attempts = statuses.len();
@@ -71,7 +80,10 @@ async fn retries_503_within_budget_but_not_4xx() {
Some(delay) => response.insert_header("retry-after", delay),
None => response.insert_header("x-ms-retry-after-ms", "0"),
};
response.insert_header("x-ms-error-code", azure_code)
match azure_code {
Some(code) => response.insert_header("x-ms-error-code", code),
None => response,
}
},
attempts as u64,
u64::from(succeeds),
@@ -97,7 +109,7 @@ async fn retries_503_within_budget_but_not_4xx() {
if let Err(error) = result {
let message = error.to_string();
assert!(message.contains(&format!("azure_request_id=request-{}", attempts - 1)));
assert!(message.contains(azure_code));
assert!(message.contains(azure_code.unwrap_or("missing")));
assert!(!format!("{error:?}").contains("sig=secret"));
}
let requests = server.received_requests().await.expect("requests");