fix(plugin): retry transient download drops in marketplace installs
This commit is contained in:
1 parent
5887722a49
commit
56b887d346
4 files changed
+405
-54
No files matched your search
@@ -14,9 +14,12 @@ downloads = ["dep:reqwest", "dep:futures", "dep:serde"]
|
||||
[dependencies]
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls", "socks"], optional = true }
|
||||
futures = { version = "0.3", optional = true }
|
||||
tokio = { version = "1", features = ["process"] }
|
||||
tokio = { version = "1", features = ["process", "time"] }
|
||||
serde = { version = "1.0", features = ["derive"], optional = true }
|
||||
uuid = { version = "1", features = ["v4"], optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { version = "1", features = ["rt"] }
|
||||
|
||||
[target.'cfg(all(unix, not(target_os = "redox")))'.dependencies]
|
||||
nix = { version = "0.31.3", features = ["user"] }
|
||||
@@ -41,6 +41,7 @@ pub fn download_candidate_urls(github_url: &str, r2_path: &str) -> Vec<String> {
|
||||
}
|
||||
|
||||
use std::pin::Pin;
|
||||
use std::time::Duration;
|
||||
|
||||
type ResponseFuture = Pin<Box<dyn std::future::Future<Output = Result<reqwest::Response, String>> + Send>>;
|
||||
|
||||
@@ -84,9 +85,262 @@ pub async fn race_download_urls(
|
||||
}
|
||||
}
|
||||
|
||||
const RESILIENT_RETRY_BASE_DELAY: Duration = Duration::from_millis(500);
|
||||
const RESILIENT_RETRY_MAX_DELAY: Duration = Duration::from_millis(4000);
|
||||
|
||||
/// Total attempts made by [`resilient_download_bytes`] for callers without a
|
||||
/// custom budget.
|
||||
pub const RESILIENT_DOWNLOAD_ATTEMPTS: usize = 4;
|
||||
|
||||
enum DownloadAttemptError {
|
||||
/// Transient: connection dropped mid-body, connect/send error, 5xx. Bytes
|
||||
/// received so far are kept and the next attempt resumes with a Range
|
||||
/// request.
|
||||
Retryable(String),
|
||||
/// Deterministic: size cap exceeded or a plain 4xx. Retrying cannot help.
|
||||
Fatal(String),
|
||||
}
|
||||
|
||||
enum ResumePlan {
|
||||
/// Append incoming chunks to the bytes already received.
|
||||
Append,
|
||||
/// The partial bytes are stale (the server ignored Range and restarted
|
||||
/// the body at zero): discard them so the body is rebuilt from zero.
|
||||
Restart,
|
||||
}
|
||||
|
||||
fn retry_delay(attempt: usize) -> Duration {
|
||||
// attempt is 1-based; the first retry (attempt == 2) waits one base step.
|
||||
let steps = 1_u32 << (attempt.saturating_sub(2)).min(16);
|
||||
RESILIENT_RETRY_BASE_DELAY.saturating_mul(steps).min(RESILIENT_RETRY_MAX_DELAY)
|
||||
}
|
||||
|
||||
enum ClassifyError {
|
||||
/// Deterministic (plain 4xx): retrying cannot help.
|
||||
FatalStatus(reqwest::StatusCode),
|
||||
/// Transient (5xx, 408, 429): a further attempt may succeed.
|
||||
RetryableStatus(reqwest::StatusCode),
|
||||
/// Transient, and the buffered bytes no longer line up with the resource
|
||||
/// (416 after a ranged request): discard them before the next attempt.
|
||||
StaleResume(reqwest::StatusCode),
|
||||
/// The server returned a partial response that cannot safely be appended
|
||||
/// to the buffered representation.
|
||||
InvalidContentRange,
|
||||
}
|
||||
|
||||
fn complete_content_range_total(headers: &reqwest::header::HeaderMap, expected_start: u64) -> Option<u64> {
|
||||
headers
|
||||
.get(reqwest::header::CONTENT_RANGE)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.strip_prefix("bytes "))
|
||||
.and_then(|value| value.split_once('/'))
|
||||
.and_then(|(range, total)| {
|
||||
let (start, end) = range.split_once('-')?;
|
||||
Some((start.parse::<u64>().ok()?, end.parse::<u64>().ok()?, total.parse::<u64>().ok()?))
|
||||
})
|
||||
.and_then(|(start, end, total)| (start == expected_start && end.checked_add(1) == Some(total)).then_some(total))
|
||||
}
|
||||
|
||||
fn has_complete_content_range(headers: &reqwest::header::HeaderMap, expected_start: u64) -> bool {
|
||||
complete_content_range_total(headers, expected_start).is_some()
|
||||
}
|
||||
|
||||
fn strong_etag(headers: &reqwest::header::HeaderMap) -> Option<reqwest::header::HeaderValue> {
|
||||
let etag = headers.get(reqwest::header::ETAG)?.clone();
|
||||
(!etag.to_str().ok()?.starts_with("W/")).then_some(etag)
|
||||
}
|
||||
|
||||
fn classify_response(
|
||||
status: reqwest::StatusCode,
|
||||
headers: &reqwest::header::HeaderMap,
|
||||
resume_from: u64,
|
||||
) -> Result<ResumePlan, ClassifyError> {
|
||||
if resume_from > 0 && status == reqwest::StatusCode::RANGE_NOT_SATISFIABLE {
|
||||
return Err(ClassifyError::StaleResume(status));
|
||||
}
|
||||
if !status.is_success() {
|
||||
return Err(match status.as_u16() {
|
||||
408 | 429 => ClassifyError::RetryableStatus(status),
|
||||
code if (400..500).contains(&code) => ClassifyError::FatalStatus(status),
|
||||
_ => ClassifyError::RetryableStatus(status),
|
||||
});
|
||||
}
|
||||
if resume_from > 0 && status == reqwest::StatusCode::OK {
|
||||
// Server ignored the Range header and restarted the body at zero.
|
||||
return Ok(ResumePlan::Restart);
|
||||
}
|
||||
if status == reqwest::StatusCode::PARTIAL_CONTENT
|
||||
&& (resume_from == 0 || !has_complete_content_range(headers, resume_from))
|
||||
{
|
||||
return Err(ClassifyError::InvalidContentRange);
|
||||
}
|
||||
Ok(ResumePlan::Append)
|
||||
}
|
||||
|
||||
/// Downloads an HTTP response body into memory, surviving transient transport
|
||||
/// failures that kill a single-connection download: connect/send errors,
|
||||
/// mid-body drops ("error decoding response body"), 5xx and 408/429 statuses.
|
||||
///
|
||||
/// Bytes received before a mid-body drop are reused only when the initial
|
||||
/// response includes a strong ETag. The next attempt sends a conditional Range
|
||||
/// request from the received offset; a server that answers 206 with the
|
||||
/// matching complete Content-Range appends, one that answers plain 200 restarts
|
||||
/// cleanly, and a 416 discards the partial bytes and retries without a Range
|
||||
/// header. Requests force identity encoding so the Range offsets and buffered
|
||||
/// bytes always refer to the same representation.
|
||||
///
|
||||
/// Error copy matches the previous single-attempt behavior (`Failed to
|
||||
/// download/read {label} from {url}: …`) so UI strings stay stable; the last
|
||||
/// retryable failure carries an `attempt N/M` suffix.
|
||||
pub async fn resilient_download_bytes(
|
||||
client: &reqwest::Client,
|
||||
url: reqwest::Url,
|
||||
label: &str,
|
||||
max_bytes: usize,
|
||||
attempts: usize,
|
||||
mut on_progress: impl FnMut(u64, Option<u64>),
|
||||
) -> Result<Vec<u8>, String> {
|
||||
let attempts = attempts.max(1);
|
||||
let mut bytes: Vec<u8> = Vec::new();
|
||||
let mut resume_validator = None;
|
||||
let mut last_error = String::new();
|
||||
for attempt in 1..=attempts {
|
||||
if attempt > 1 {
|
||||
tokio::time::sleep(retry_delay(attempt)).await;
|
||||
}
|
||||
match download_attempt(client, &url, label, max_bytes, &mut bytes, &mut resume_validator, &mut on_progress)
|
||||
.await
|
||||
{
|
||||
Ok(()) => return Ok(bytes),
|
||||
Err(DownloadAttemptError::Fatal(error)) => return Err(error),
|
||||
Err(DownloadAttemptError::Retryable(error)) => {
|
||||
last_error = format!("{error} (download attempt {attempt}/{attempts})");
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(last_error)
|
||||
}
|
||||
|
||||
async fn download_attempt(
|
||||
client: &reqwest::Client,
|
||||
url: &reqwest::Url,
|
||||
label: &str,
|
||||
max_bytes: usize,
|
||||
bytes: &mut Vec<u8>,
|
||||
resume_validator: &mut Option<reqwest::header::HeaderValue>,
|
||||
on_progress: &mut impl FnMut(u64, Option<u64>),
|
||||
) -> Result<(), DownloadAttemptError> {
|
||||
use futures::StreamExt;
|
||||
|
||||
// Without a strong validator, a later Range response could belong to a
|
||||
// newer representation at the same URL. Restart rather than risk mixing
|
||||
// the old prefix with new bytes.
|
||||
if !bytes.is_empty() && resume_validator.is_none() {
|
||||
bytes.clear();
|
||||
}
|
||||
let resume_from = bytes.len() as u64;
|
||||
let mut request = client.get(url.clone()).header(reqwest::header::ACCEPT_ENCODING, "identity");
|
||||
if resume_from > 0 {
|
||||
request = request.header(reqwest::header::RANGE, format!("bytes={resume_from}-"));
|
||||
request = request.header(
|
||||
reqwest::header::IF_RANGE,
|
||||
resume_validator.as_ref().expect("partial bytes have a validator").clone(),
|
||||
);
|
||||
}
|
||||
let response = request
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| DownloadAttemptError::Retryable(format!("Failed to download {label} from {url}: {error}")))?;
|
||||
let status_error = |status: reqwest::StatusCode| format!("Failed to download {label} from {url}: HTTP {status}");
|
||||
match classify_response(response.status(), response.headers(), resume_from) {
|
||||
Ok(ResumePlan::Append) => {}
|
||||
Ok(ResumePlan::Restart) => {
|
||||
bytes.clear();
|
||||
*resume_validator = strong_etag(response.headers());
|
||||
}
|
||||
// The buffered bytes no longer line up with the resource: drop them so
|
||||
// the next attempt re-requests without a Range header. Never read the
|
||||
// 416 body — it is an error document, not content.
|
||||
Err(ClassifyError::StaleResume(status)) => {
|
||||
bytes.clear();
|
||||
*resume_validator = None;
|
||||
return Err(DownloadAttemptError::Retryable(status_error(status)));
|
||||
}
|
||||
Err(ClassifyError::InvalidContentRange) => {
|
||||
bytes.clear();
|
||||
*resume_validator = None;
|
||||
return Err(DownloadAttemptError::Retryable(format!(
|
||||
"Failed to download {label} from {url}: invalid Content-Range for resumed response"
|
||||
)));
|
||||
}
|
||||
Err(ClassifyError::RetryableStatus(status)) => {
|
||||
return Err(DownloadAttemptError::Retryable(status_error(status)));
|
||||
}
|
||||
Err(ClassifyError::FatalStatus(status)) => {
|
||||
return Err(DownloadAttemptError::Fatal(status_error(status)));
|
||||
}
|
||||
}
|
||||
if resume_from == 0 {
|
||||
*resume_validator = strong_etag(response.headers());
|
||||
}
|
||||
if response.content_length().is_some_and(|length| bytes.len() as u64 + length > max_bytes as u64) {
|
||||
return Err(DownloadAttemptError::Fatal(format!("{label} exceeds {max_bytes} bytes")));
|
||||
}
|
||||
let expected_total = (response.status() == reqwest::StatusCode::PARTIAL_CONTENT)
|
||||
.then(|| complete_content_range_total(response.headers(), resume_from))
|
||||
.flatten();
|
||||
let total = response.content_length().map(|length| bytes.len() as u64 + length);
|
||||
let mut stream = response.bytes_stream();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk
|
||||
.map_err(|error| DownloadAttemptError::Retryable(format!("Failed to read {label} from {url}: {error}")))?;
|
||||
if bytes.len().saturating_add(chunk.len()) > max_bytes {
|
||||
return Err(DownloadAttemptError::Fatal(format!("{label} exceeds {max_bytes} bytes")));
|
||||
}
|
||||
bytes.extend_from_slice(&chunk);
|
||||
on_progress(bytes.len() as u64, total);
|
||||
}
|
||||
if expected_total.is_some_and(|total| bytes.len() as u64 != total) {
|
||||
return Err(DownloadAttemptError::Retryable(format!(
|
||||
"Failed to read {label} from {url}: incomplete Content-Range body"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{download_candidate_urls, DownloadSource};
|
||||
use super::{classify_response, has_complete_content_range, retry_delay, strong_etag, ClassifyError, ResumePlan};
|
||||
use super::{download_candidate_urls, resilient_download_bytes, DownloadSource};
|
||||
use reqwest::header::{HeaderMap, HeaderValue, CONTENT_RANGE, ETAG};
|
||||
use std::io::{Read, Write};
|
||||
use std::net::{Shutdown, TcpListener};
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
fn status(code: u16) -> reqwest::StatusCode {
|
||||
reqwest::StatusCode::from_u16(code).unwrap()
|
||||
}
|
||||
|
||||
fn headers(entries: &[(&reqwest::header::HeaderName, &str)]) -> HeaderMap {
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in entries {
|
||||
headers.insert((*name).clone(), HeaderValue::from_str(value).unwrap());
|
||||
}
|
||||
headers
|
||||
}
|
||||
|
||||
fn read_request_headers(stream: &mut std::net::TcpStream) -> String {
|
||||
stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap();
|
||||
let mut request = Vec::new();
|
||||
let mut buffer = [0; 1024];
|
||||
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
let read = stream.read(&mut buffer).unwrap();
|
||||
assert_ne!(read, 0, "client closed connection before sending headers");
|
||||
request.extend_from_slice(&buffer[..read]);
|
||||
}
|
||||
String::from_utf8(request).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn download_candidates_exclude_third_party_github_proxy() {
|
||||
@@ -115,4 +369,122 @@ mod tests {
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_appends_fresh_and_partial_content_responses() {
|
||||
assert!(matches!(classify_response(status(200), &HeaderMap::new(), 0), Ok(ResumePlan::Append)));
|
||||
let partial = headers(&[(&CONTENT_RANGE, "bytes 1024-4095/4096")]);
|
||||
assert!(matches!(classify_response(status(206), &partial, 1024), Ok(ResumePlan::Append)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_restarts_when_server_ignores_range() {
|
||||
assert!(matches!(classify_response(status(200), &HeaderMap::new(), 1024), Ok(ResumePlan::Restart)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_fails_unsatisfiable_range_as_stale_resume() {
|
||||
// 416 is an error status: the attempt must fail (with the buffered
|
||||
// bytes dropped by the caller) instead of reading the error body.
|
||||
assert!(matches!(classify_response(status(416), &HeaderMap::new(), 1024), Err(ClassifyError::StaleResume(_))));
|
||||
// Without a resume offset a 416 is just a deterministic client error.
|
||||
assert!(matches!(classify_response(status(416), &HeaderMap::new(), 0), Err(ClassifyError::FatalStatus(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_fails_deterministic_client_errors_without_retry() {
|
||||
assert!(matches!(classify_response(status(404), &HeaderMap::new(), 0), Err(ClassifyError::FatalStatus(_))));
|
||||
assert!(matches!(classify_response(status(403), &HeaderMap::new(), 0), Err(ClassifyError::FatalStatus(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_retries_transient_statuses() {
|
||||
assert!(matches!(classify_response(status(408), &HeaderMap::new(), 0), Err(ClassifyError::RetryableStatus(_))));
|
||||
assert!(matches!(classify_response(status(429), &HeaderMap::new(), 0), Err(ClassifyError::RetryableStatus(_))));
|
||||
assert!(matches!(classify_response(status(500), &HeaderMap::new(), 0), Err(ClassifyError::RetryableStatus(_))));
|
||||
assert!(matches!(classify_response(status(502), &HeaderMap::new(), 0), Err(ClassifyError::RetryableStatus(_))));
|
||||
assert!(matches!(classify_response(status(503), &HeaderMap::new(), 0), Err(ClassifyError::RetryableStatus(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_partial_responses_that_cannot_be_safely_appended() {
|
||||
let wrong_start = headers(&[(&CONTENT_RANGE, "bytes 0-1023/4096")]);
|
||||
assert!(!has_complete_content_range(&wrong_start, 1024));
|
||||
assert!(matches!(classify_response(status(206), &wrong_start, 1024), Err(ClassifyError::InvalidContentRange)));
|
||||
|
||||
let initial_partial = headers(&[(&CONTENT_RANGE, "bytes 0-1023/4096")]);
|
||||
assert!(matches!(classify_response(status(206), &initial_partial, 0), Err(ClassifyError::InvalidContentRange)));
|
||||
|
||||
let incomplete_suffix = headers(&[(&CONTENT_RANGE, "bytes 1024-2047/4096")]);
|
||||
assert!(matches!(
|
||||
classify_response(status(206), &incomplete_suffix, 1024),
|
||||
Err(ClassifyError::InvalidContentRange)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_strong_etags_enable_resume() {
|
||||
let strong = headers(&[(&ETAG, "\"release-1\"")]);
|
||||
assert_eq!(strong_etag(&strong).as_ref().map(HeaderValue::as_bytes), Some(b"\"release-1\"".as_slice()));
|
||||
|
||||
let weak = headers(&[(&ETAG, "W/\"release-1\"")]);
|
||||
assert!(strong_etag(&weak).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resumes_only_when_the_etag_matches_the_partial_response() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = thread::spawn(move || {
|
||||
let (mut initial, _) = listener.accept().unwrap();
|
||||
let initial_request = read_request_headers(&mut initial).to_ascii_lowercase();
|
||||
assert!(!initial_request.contains("range:"));
|
||||
assert!(initial_request.contains("accept-encoding: identity"), "{initial_request}");
|
||||
initial
|
||||
.write_all(
|
||||
b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\nETag: \"release-1\"\r\nConnection: close\r\n\r\nold-",
|
||||
)
|
||||
.unwrap();
|
||||
initial.shutdown(Shutdown::Both).unwrap();
|
||||
|
||||
let (mut resumed, _) = listener.accept().unwrap();
|
||||
let resumed_request = read_request_headers(&mut resumed).to_ascii_lowercase();
|
||||
assert!(resumed_request.contains("range: bytes=4-"), "{resumed_request}");
|
||||
assert!(resumed_request.contains("if-range: \"release-1\""), "{resumed_request}");
|
||||
assert!(resumed_request.contains("accept-encoding: identity"), "{resumed_request}");
|
||||
// The resource changed after the interrupted first response. A
|
||||
// compliant server ignores Range for a mismatched If-Range and
|
||||
// sends the new full representation.
|
||||
resumed
|
||||
.write_all(
|
||||
b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\nETag: \"release-2\"\r\nConnection: close\r\n\r\nnew-body",
|
||||
)
|
||||
.unwrap();
|
||||
});
|
||||
|
||||
let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap();
|
||||
let client = reqwest::Client::new();
|
||||
let downloaded = runtime
|
||||
.block_on(resilient_download_bytes(
|
||||
&client,
|
||||
format!("http://{address}/artifact").parse().unwrap(),
|
||||
"test artifact",
|
||||
1024,
|
||||
2,
|
||||
|_, _| {},
|
||||
))
|
||||
.unwrap();
|
||||
server.join().unwrap();
|
||||
|
||||
assert_eq!(downloaded, b"new-body");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_delay_backs_off_then_caps() {
|
||||
assert_eq!(retry_delay(2), Duration::from_millis(500));
|
||||
assert_eq!(retry_delay(3), Duration::from_millis(1000));
|
||||
assert_eq!(retry_delay(4), Duration::from_millis(2000));
|
||||
assert_eq!(retry_delay(5), Duration::from_millis(4000));
|
||||
assert_eq!(retry_delay(20), Duration::from_millis(4000));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user