mirror of
https://github.com/openai/codex.git
synced 2026-11-22 15:15:18 +00:00
Make diagnostic report uploads resilient to slow networks (#42096)
## Why Diagnostic reports can span several envelopes, and the previous 10-second shared network budget could expire before slow uploads and their attachments completed. ## What changed - Give each diagnostic report a single five-minute deadline shared by the event, attachments, retries, and retry backoff. - Stop reading or sending later attachments once the deadline or a Sentry rate limit is reached. - Limit the app server to three concurrent report uploads and return an overloaded JSON-RPC error for additional requests until a slot is released. ## Testing Add coverage for slow multi-envelope reports, deadline-aware retries, skipped attachments, rate-limit handling, and concurrency-slot release after failures. GitOrigin-RevId: bdacf6c9731df16d2763e204565986eb6d540233
This commit is contained in:
1 parent
ade0ccacf9
commit
a17ee5705c
4 files changed
+210
-46
No files matched your search
@@ -46,24 +46,21 @@ pub(super) fn gzip_envelope_request(
|
||||
.timeout(timeout)
|
||||
}
|
||||
|
||||
/// Send an already-serialized envelope, charging requests and backoff to the shared budget.
|
||||
/// Send an already-serialized envelope within the report's shared deadline.
|
||||
pub(super) async fn send_gzip_envelope(
|
||||
client_pool: &RouteAwareClientPool,
|
||||
dsn: &Dsn,
|
||||
body: Vec<u8>,
|
||||
kind: EnvelopeKind,
|
||||
remaining_upload_time: &mut Duration,
|
||||
deadline: Instant,
|
||||
rate_limited: &mut bool,
|
||||
) -> Result<StatusCode> {
|
||||
let request_started_at = Instant::now();
|
||||
let body = Bytes::from(body);
|
||||
// Retain the exact gzip bytes for at most two diagnostic retries. Never replay the core.
|
||||
let mut retry_delays = [250, 500].map(Duration::from_millis).into_iter();
|
||||
let response = loop {
|
||||
let timeout = remaining_upload_time.saturating_sub(request_started_at.elapsed());
|
||||
if timeout.is_zero() {
|
||||
*remaining_upload_time = Duration::ZERO;
|
||||
anyhow::bail!("feedback upload network wait budget exhausted");
|
||||
}
|
||||
let timeout = deadline.saturating_duration_since(Instant::now());
|
||||
anyhow::ensure!(!timeout.is_zero(), "feedback upload deadline exceeded");
|
||||
let response = gzip_envelope_request(client_pool, dsn, body.clone(), timeout)
|
||||
.send()
|
||||
.await;
|
||||
@@ -77,7 +74,7 @@ pub(super) async fn send_gzip_envelope(
|
||||
}) {
|
||||
// Even accepted requests can impose a quota on events or attachments.
|
||||
// https://develop.sentry.dev/sdk/foundations/transport/rate-limiting/
|
||||
*remaining_upload_time = Duration::ZERO;
|
||||
*rate_limited = true;
|
||||
break response;
|
||||
}
|
||||
let retryable = match &response {
|
||||
@@ -96,9 +93,8 @@ pub(super) async fn send_gzip_envelope(
|
||||
response.status() == StatusCode::TOO_MANY_REQUESTS
|
||||
&& (retry_delay.is_none() || !response.headers().contains_key("Retry-After"))
|
||||
}) {
|
||||
// Bare 429s default to a cooldown longer than this upload budget.
|
||||
// Never let later files bypass an exhausted rate limit.
|
||||
*remaining_upload_time = Duration::ZERO;
|
||||
*rate_limited = true;
|
||||
break response;
|
||||
}
|
||||
let Some(retry_delay) = retry_delay else {
|
||||
@@ -111,20 +107,16 @@ pub(super) async fn send_gzip_envelope(
|
||||
.map(|value| {
|
||||
// An invalid cooldown is not permission to send immediately.
|
||||
parse_retry_after(value.to_str().unwrap_or_default())
|
||||
.unwrap_or(*remaining_upload_time)
|
||||
.unwrap_or_else(|| deadline.saturating_duration_since(Instant::now()))
|
||||
});
|
||||
let delay = retry_delay.max(retry_after.unwrap_or_default());
|
||||
if delay >= remaining_upload_time.saturating_sub(request_started_at.elapsed()) {
|
||||
// The cooldown applies to later files too, not just this retry.
|
||||
*remaining_upload_time = Duration::ZERO;
|
||||
if delay >= deadline.saturating_duration_since(Instant::now()) {
|
||||
// Leave long cooldowns for a later submission, including its remaining files.
|
||||
*rate_limited = true;
|
||||
break response;
|
||||
}
|
||||
tokio::time::sleep(delay).await;
|
||||
if request_started_at.elapsed() >= *remaining_upload_time {
|
||||
break response;
|
||||
}
|
||||
};
|
||||
*remaining_upload_time = remaining_upload_time.saturating_sub(request_started_at.elapsed());
|
||||
response
|
||||
.map(|response| response.status())
|
||||
.context("failed to upload feedback to Sentry")
|
||||
|
||||
Reference in new issue
Block a user