From 56b887d3461bf1ce4d2a2a6a70af698e07a03a0a Mon Sep 17 00:00:00 2001 From: Abeautifulsnow <1491444340@qq.com> Date: Sun, 4 Oct 2026 03:13:16 +0800 Subject: [PATCH] fix(plugin): retry transient download drops in marketplace installs --- crates/dbx-platform/Cargo.toml | 5 +- crates/dbx-platform/src/download.rs | 374 +++++++++++++++++- crates/dbx-plugin-runtime/Cargo.toml | 2 +- .../src/plugins/marketplace.rs | 78 ++-- 4 files changed, 405 insertions(+), 54 deletions(-) diff --git a/crates/dbx-platform/Cargo.toml b/crates/dbx-platform/Cargo.toml index 1bfc22ea8a..40d8d37b05 100644 --- a/crates/dbx-platform/Cargo.toml +++ b/crates/dbx-platform/Cargo.toml @@ -14,9 +14,12 @@ downloads = ["dep:reqwest", "dep:futures", "dep:serde"] [dependencies] reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls", "socks"], optional = true } futures = { version = "0.3", optional = true } -tokio = { version = "1", features = ["process"] } +tokio = { version = "1", features = ["process", "time"] } serde = { version = "1.0", features = ["derive"], optional = true } uuid = { version = "1", features = ["v4"], optional = true } +[dev-dependencies] +tokio = { version = "1", features = ["rt"] } + [target.'cfg(all(unix, not(target_os = "redox")))'.dependencies] nix = { version = "0.31.3", features = ["user"] } diff --git a/crates/dbx-platform/src/download.rs b/crates/dbx-platform/src/download.rs index ad5a7e880d..9cc8a7369c 100644 --- a/crates/dbx-platform/src/download.rs +++ b/crates/dbx-platform/src/download.rs @@ -41,6 +41,7 @@ pub fn download_candidate_urls(github_url: &str, r2_path: &str) -> Vec { } use std::pin::Pin; +use std::time::Duration; type ResponseFuture = Pin> + Send>>; @@ -84,9 +85,262 @@ pub async fn race_download_urls( } } +const RESILIENT_RETRY_BASE_DELAY: Duration = Duration::from_millis(500); +const RESILIENT_RETRY_MAX_DELAY: Duration = Duration::from_millis(4000); + +/// Total attempts made by [`resilient_download_bytes`] for callers without a +/// custom budget. +pub const RESILIENT_DOWNLOAD_ATTEMPTS: usize = 4; + +enum DownloadAttemptError { + /// Transient: connection dropped mid-body, connect/send error, 5xx. Bytes + /// received so far are kept and the next attempt resumes with a Range + /// request. + Retryable(String), + /// Deterministic: size cap exceeded or a plain 4xx. Retrying cannot help. + Fatal(String), +} + +enum ResumePlan { + /// Append incoming chunks to the bytes already received. + Append, + /// The partial bytes are stale (the server ignored Range and restarted + /// the body at zero): discard them so the body is rebuilt from zero. + Restart, +} + +fn retry_delay(attempt: usize) -> Duration { + // attempt is 1-based; the first retry (attempt == 2) waits one base step. + let steps = 1_u32 << (attempt.saturating_sub(2)).min(16); + RESILIENT_RETRY_BASE_DELAY.saturating_mul(steps).min(RESILIENT_RETRY_MAX_DELAY) +} + +enum ClassifyError { + /// Deterministic (plain 4xx): retrying cannot help. + FatalStatus(reqwest::StatusCode), + /// Transient (5xx, 408, 429): a further attempt may succeed. + RetryableStatus(reqwest::StatusCode), + /// Transient, and the buffered bytes no longer line up with the resource + /// (416 after a ranged request): discard them before the next attempt. + StaleResume(reqwest::StatusCode), + /// The server returned a partial response that cannot safely be appended + /// to the buffered representation. + InvalidContentRange, +} + +fn complete_content_range_total(headers: &reqwest::header::HeaderMap, expected_start: u64) -> Option { + headers + .get(reqwest::header::CONTENT_RANGE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.strip_prefix("bytes ")) + .and_then(|value| value.split_once('/')) + .and_then(|(range, total)| { + let (start, end) = range.split_once('-')?; + Some((start.parse::().ok()?, end.parse::().ok()?, total.parse::().ok()?)) + }) + .and_then(|(start, end, total)| (start == expected_start && end.checked_add(1) == Some(total)).then_some(total)) +} + +fn has_complete_content_range(headers: &reqwest::header::HeaderMap, expected_start: u64) -> bool { + complete_content_range_total(headers, expected_start).is_some() +} + +fn strong_etag(headers: &reqwest::header::HeaderMap) -> Option { + let etag = headers.get(reqwest::header::ETAG)?.clone(); + (!etag.to_str().ok()?.starts_with("W/")).then_some(etag) +} + +fn classify_response( + status: reqwest::StatusCode, + headers: &reqwest::header::HeaderMap, + resume_from: u64, +) -> Result { + if resume_from > 0 && status == reqwest::StatusCode::RANGE_NOT_SATISFIABLE { + return Err(ClassifyError::StaleResume(status)); + } + if !status.is_success() { + return Err(match status.as_u16() { + 408 | 429 => ClassifyError::RetryableStatus(status), + code if (400..500).contains(&code) => ClassifyError::FatalStatus(status), + _ => ClassifyError::RetryableStatus(status), + }); + } + if resume_from > 0 && status == reqwest::StatusCode::OK { + // Server ignored the Range header and restarted the body at zero. + return Ok(ResumePlan::Restart); + } + if status == reqwest::StatusCode::PARTIAL_CONTENT + && (resume_from == 0 || !has_complete_content_range(headers, resume_from)) + { + return Err(ClassifyError::InvalidContentRange); + } + Ok(ResumePlan::Append) +} + +/// Downloads an HTTP response body into memory, surviving transient transport +/// failures that kill a single-connection download: connect/send errors, +/// mid-body drops ("error decoding response body"), 5xx and 408/429 statuses. +/// +/// Bytes received before a mid-body drop are reused only when the initial +/// response includes a strong ETag. The next attempt sends a conditional Range +/// request from the received offset; a server that answers 206 with the +/// matching complete Content-Range appends, one that answers plain 200 restarts +/// cleanly, and a 416 discards the partial bytes and retries without a Range +/// header. Requests force identity encoding so the Range offsets and buffered +/// bytes always refer to the same representation. +/// +/// Error copy matches the previous single-attempt behavior (`Failed to +/// download/read {label} from {url}: …`) so UI strings stay stable; the last +/// retryable failure carries an `attempt N/M` suffix. +pub async fn resilient_download_bytes( + client: &reqwest::Client, + url: reqwest::Url, + label: &str, + max_bytes: usize, + attempts: usize, + mut on_progress: impl FnMut(u64, Option), +) -> Result, String> { + let attempts = attempts.max(1); + let mut bytes: Vec = Vec::new(); + let mut resume_validator = None; + let mut last_error = String::new(); + for attempt in 1..=attempts { + if attempt > 1 { + tokio::time::sleep(retry_delay(attempt)).await; + } + match download_attempt(client, &url, label, max_bytes, &mut bytes, &mut resume_validator, &mut on_progress) + .await + { + Ok(()) => return Ok(bytes), + Err(DownloadAttemptError::Fatal(error)) => return Err(error), + Err(DownloadAttemptError::Retryable(error)) => { + last_error = format!("{error} (download attempt {attempt}/{attempts})"); + } + } + } + Err(last_error) +} + +async fn download_attempt( + client: &reqwest::Client, + url: &reqwest::Url, + label: &str, + max_bytes: usize, + bytes: &mut Vec, + resume_validator: &mut Option, + on_progress: &mut impl FnMut(u64, Option), +) -> Result<(), DownloadAttemptError> { + use futures::StreamExt; + + // Without a strong validator, a later Range response could belong to a + // newer representation at the same URL. Restart rather than risk mixing + // the old prefix with new bytes. + if !bytes.is_empty() && resume_validator.is_none() { + bytes.clear(); + } + let resume_from = bytes.len() as u64; + let mut request = client.get(url.clone()).header(reqwest::header::ACCEPT_ENCODING, "identity"); + if resume_from > 0 { + request = request.header(reqwest::header::RANGE, format!("bytes={resume_from}-")); + request = request.header( + reqwest::header::IF_RANGE, + resume_validator.as_ref().expect("partial bytes have a validator").clone(), + ); + } + let response = request + .send() + .await + .map_err(|error| DownloadAttemptError::Retryable(format!("Failed to download {label} from {url}: {error}")))?; + let status_error = |status: reqwest::StatusCode| format!("Failed to download {label} from {url}: HTTP {status}"); + match classify_response(response.status(), response.headers(), resume_from) { + Ok(ResumePlan::Append) => {} + Ok(ResumePlan::Restart) => { + bytes.clear(); + *resume_validator = strong_etag(response.headers()); + } + // The buffered bytes no longer line up with the resource: drop them so + // the next attempt re-requests without a Range header. Never read the + // 416 body — it is an error document, not content. + Err(ClassifyError::StaleResume(status)) => { + bytes.clear(); + *resume_validator = None; + return Err(DownloadAttemptError::Retryable(status_error(status))); + } + Err(ClassifyError::InvalidContentRange) => { + bytes.clear(); + *resume_validator = None; + return Err(DownloadAttemptError::Retryable(format!( + "Failed to download {label} from {url}: invalid Content-Range for resumed response" + ))); + } + Err(ClassifyError::RetryableStatus(status)) => { + return Err(DownloadAttemptError::Retryable(status_error(status))); + } + Err(ClassifyError::FatalStatus(status)) => { + return Err(DownloadAttemptError::Fatal(status_error(status))); + } + } + if resume_from == 0 { + *resume_validator = strong_etag(response.headers()); + } + if response.content_length().is_some_and(|length| bytes.len() as u64 + length > max_bytes as u64) { + return Err(DownloadAttemptError::Fatal(format!("{label} exceeds {max_bytes} bytes"))); + } + let expected_total = (response.status() == reqwest::StatusCode::PARTIAL_CONTENT) + .then(|| complete_content_range_total(response.headers(), resume_from)) + .flatten(); + let total = response.content_length().map(|length| bytes.len() as u64 + length); + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk + .map_err(|error| DownloadAttemptError::Retryable(format!("Failed to read {label} from {url}: {error}")))?; + if bytes.len().saturating_add(chunk.len()) > max_bytes { + return Err(DownloadAttemptError::Fatal(format!("{label} exceeds {max_bytes} bytes"))); + } + bytes.extend_from_slice(&chunk); + on_progress(bytes.len() as u64, total); + } + if expected_total.is_some_and(|total| bytes.len() as u64 != total) { + return Err(DownloadAttemptError::Retryable(format!( + "Failed to read {label} from {url}: incomplete Content-Range body" + ))); + } + Ok(()) +} + #[cfg(test)] mod tests { - use super::{download_candidate_urls, DownloadSource}; + use super::{classify_response, has_complete_content_range, retry_delay, strong_etag, ClassifyError, ResumePlan}; + use super::{download_candidate_urls, resilient_download_bytes, DownloadSource}; + use reqwest::header::{HeaderMap, HeaderValue, CONTENT_RANGE, ETAG}; + use std::io::{Read, Write}; + use std::net::{Shutdown, TcpListener}; + use std::thread; + use std::time::Duration; + + fn status(code: u16) -> reqwest::StatusCode { + reqwest::StatusCode::from_u16(code).unwrap() + } + + fn headers(entries: &[(&reqwest::header::HeaderName, &str)]) -> HeaderMap { + let mut headers = HeaderMap::new(); + for (name, value) in entries { + headers.insert((*name).clone(), HeaderValue::from_str(value).unwrap()); + } + headers + } + + fn read_request_headers(stream: &mut std::net::TcpStream) -> String { + stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + let mut request = Vec::new(); + let mut buffer = [0; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let read = stream.read(&mut buffer).unwrap(); + assert_ne!(read, 0, "client closed connection before sending headers"); + request.extend_from_slice(&buffer[..read]); + } + String::from_utf8(request).unwrap() + } #[test] fn download_candidates_exclude_third_party_github_proxy() { @@ -115,4 +369,122 @@ mod tests { ] ); } + + #[test] + fn classify_appends_fresh_and_partial_content_responses() { + assert!(matches!(classify_response(status(200), &HeaderMap::new(), 0), Ok(ResumePlan::Append))); + let partial = headers(&[(&CONTENT_RANGE, "bytes 1024-4095/4096")]); + assert!(matches!(classify_response(status(206), &partial, 1024), Ok(ResumePlan::Append))); + } + + #[test] + fn classify_restarts_when_server_ignores_range() { + assert!(matches!(classify_response(status(200), &HeaderMap::new(), 1024), Ok(ResumePlan::Restart))); + } + + #[test] + fn classify_fails_unsatisfiable_range_as_stale_resume() { + // 416 is an error status: the attempt must fail (with the buffered + // bytes dropped by the caller) instead of reading the error body. + assert!(matches!(classify_response(status(416), &HeaderMap::new(), 1024), Err(ClassifyError::StaleResume(_)))); + // Without a resume offset a 416 is just a deterministic client error. + assert!(matches!(classify_response(status(416), &HeaderMap::new(), 0), Err(ClassifyError::FatalStatus(_)))); + } + + #[test] + fn classify_fails_deterministic_client_errors_without_retry() { + assert!(matches!(classify_response(status(404), &HeaderMap::new(), 0), Err(ClassifyError::FatalStatus(_)))); + assert!(matches!(classify_response(status(403), &HeaderMap::new(), 0), Err(ClassifyError::FatalStatus(_)))); + } + + #[test] + fn classify_retries_transient_statuses() { + assert!(matches!(classify_response(status(408), &HeaderMap::new(), 0), Err(ClassifyError::RetryableStatus(_)))); + assert!(matches!(classify_response(status(429), &HeaderMap::new(), 0), Err(ClassifyError::RetryableStatus(_)))); + assert!(matches!(classify_response(status(500), &HeaderMap::new(), 0), Err(ClassifyError::RetryableStatus(_)))); + assert!(matches!(classify_response(status(502), &HeaderMap::new(), 0), Err(ClassifyError::RetryableStatus(_)))); + assert!(matches!(classify_response(status(503), &HeaderMap::new(), 0), Err(ClassifyError::RetryableStatus(_)))); + } + + #[test] + fn rejects_partial_responses_that_cannot_be_safely_appended() { + let wrong_start = headers(&[(&CONTENT_RANGE, "bytes 0-1023/4096")]); + assert!(!has_complete_content_range(&wrong_start, 1024)); + assert!(matches!(classify_response(status(206), &wrong_start, 1024), Err(ClassifyError::InvalidContentRange))); + + let initial_partial = headers(&[(&CONTENT_RANGE, "bytes 0-1023/4096")]); + assert!(matches!(classify_response(status(206), &initial_partial, 0), Err(ClassifyError::InvalidContentRange))); + + let incomplete_suffix = headers(&[(&CONTENT_RANGE, "bytes 1024-2047/4096")]); + assert!(matches!( + classify_response(status(206), &incomplete_suffix, 1024), + Err(ClassifyError::InvalidContentRange) + )); + } + + #[test] + fn only_strong_etags_enable_resume() { + let strong = headers(&[(&ETAG, "\"release-1\"")]); + assert_eq!(strong_etag(&strong).as_ref().map(HeaderValue::as_bytes), Some(b"\"release-1\"".as_slice())); + + let weak = headers(&[(&ETAG, "W/\"release-1\"")]); + assert!(strong_etag(&weak).is_none()); + } + + #[test] + fn resumes_only_when_the_etag_matches_the_partial_response() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut initial, _) = listener.accept().unwrap(); + let initial_request = read_request_headers(&mut initial).to_ascii_lowercase(); + assert!(!initial_request.contains("range:")); + assert!(initial_request.contains("accept-encoding: identity"), "{initial_request}"); + initial + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\nETag: \"release-1\"\r\nConnection: close\r\n\r\nold-", + ) + .unwrap(); + initial.shutdown(Shutdown::Both).unwrap(); + + let (mut resumed, _) = listener.accept().unwrap(); + let resumed_request = read_request_headers(&mut resumed).to_ascii_lowercase(); + assert!(resumed_request.contains("range: bytes=4-"), "{resumed_request}"); + assert!(resumed_request.contains("if-range: \"release-1\""), "{resumed_request}"); + assert!(resumed_request.contains("accept-encoding: identity"), "{resumed_request}"); + // The resource changed after the interrupted first response. A + // compliant server ignores Range for a mismatched If-Range and + // sends the new full representation. + resumed + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\nETag: \"release-2\"\r\nConnection: close\r\n\r\nnew-body", + ) + .unwrap(); + }); + + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + let client = reqwest::Client::new(); + let downloaded = runtime + .block_on(resilient_download_bytes( + &client, + format!("http://{address}/artifact").parse().unwrap(), + "test artifact", + 1024, + 2, + |_, _| {}, + )) + .unwrap(); + server.join().unwrap(); + + assert_eq!(downloaded, b"new-body"); + } + + #[test] + fn retry_delay_backs_off_then_caps() { + assert_eq!(retry_delay(2), Duration::from_millis(500)); + assert_eq!(retry_delay(3), Duration::from_millis(1000)); + assert_eq!(retry_delay(4), Duration::from_millis(2000)); + assert_eq!(retry_delay(5), Duration::from_millis(4000)); + assert_eq!(retry_delay(20), Duration::from_millis(4000)); + } } diff --git a/crates/dbx-plugin-runtime/Cargo.toml b/crates/dbx-plugin-runtime/Cargo.toml index 70cc071529..6afd2c0d99 100644 --- a/crates/dbx-plugin-runtime/Cargo.toml +++ b/crates/dbx-plugin-runtime/Cargo.toml @@ -10,7 +10,7 @@ default = [] test-support = [] [dependencies] -dbx-platform = { path = "../dbx-platform", features = ["host-prompts"] } +dbx-platform = { path = "../dbx-platform", features = ["host-prompts", "downloads"] } dbx-types = { path = "../dbx-types", default-features = false } serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } diff --git a/crates/dbx-plugin-runtime/src/plugins/marketplace.rs b/crates/dbx-plugin-runtime/src/plugins/marketplace.rs index fdf8aabe59..bb20ade844 100644 --- a/crates/dbx-plugin-runtime/src/plugins/marketplace.rs +++ b/crates/dbx-plugin-runtime/src/plugins/marketplace.rs @@ -5,7 +5,6 @@ use std::path::{Path, PathBuf}; use std::time::Duration; use fs2::FileExt; -use futures::StreamExt; use reqwest::redirect::Policy; use reqwest::{Client, Url}; use semver::Version; @@ -364,7 +363,7 @@ impl PluginMarketplace { validate_repository(repository)?; let primary_url = repository_catalog_url(repository)?; let (raw, catalog_url) = match self - .download_limited(primary_url.clone(), MAX_PLUGIN_CATALOG_BYTES, "Plugin catalog") + .download_limited(primary_url.clone(), MAX_PLUGIN_CATALOG_BYTES, "Plugin catalog", |_, _| {}) .await { Ok(raw) => (raw, primary_url), @@ -372,7 +371,7 @@ impl PluginMarketplace { let fallback_url = Url::parse(OFFICIAL_CATALOG_FALLBACK_URL).expect("built-in catalog fallback URL is valid"); let raw = self - .download_limited(fallback_url.clone(), MAX_PLUGIN_CATALOG_BYTES, "Plugin catalog") + .download_limited(fallback_url.clone(), MAX_PLUGIN_CATALOG_BYTES, "Plugin catalog", |_, _| {}) .await .map_err(|fallback_error| { format!( @@ -430,7 +429,8 @@ impl PluginMarketplace { } let artifact_url = Url::parse(&artifact.url).map_err(|error| format!("Invalid plugin artifact URL: {error}"))?; - let package = self.download_limited(artifact_url, MAX_PLUGIN_PACKAGE_BYTES, "Plugin package").await?; + let package = + self.download_limited(artifact_url, MAX_PLUGIN_PACKAGE_BYTES, "Plugin package", |_, _| {}).await?; verify_artifact_bytes(artifact, &package)?; let trust_store = marketplace_trust_store(&self.root_dir, repository.kind)?; let expectation = PluginPackageExpectation { @@ -464,56 +464,32 @@ impl PluginMarketplace { F: FnMut(u64, Option), { let url = parse_http_url(url.trim(), "Plugin package URL")?; - let response = self - .client - .get(url.clone()) - .send() - .await - .map_err(|error| format!("Failed to download Plugin package from {url}: {error}"))?; - if !response.status().is_success() { - return Err(format!("Failed to download Plugin package from {url}: HTTP {}", response.status())); - } - if response.content_length().is_some_and(|length| length > MAX_PLUGIN_PACKAGE_BYTES as u64) { - return Err(format!("Plugin package exceeds {MAX_PLUGIN_PACKAGE_BYTES} bytes")); - } - let total = response.content_length(); - let mut bytes = Vec::new(); - let mut stream = response.bytes_stream(); - while let Some(chunk) = stream.next().await { - let chunk = chunk.map_err(|error| format!("Failed to read Plugin package from {url}: {error}"))?; - if bytes.len().saturating_add(chunk.len()) > MAX_PLUGIN_PACKAGE_BYTES { - return Err(format!("Plugin package exceeds {MAX_PLUGIN_PACKAGE_BYTES} bytes")); - } - bytes.extend_from_slice(&chunk); - on_progress(bytes.len() as u64, total); - } + let package = self.download_limited(url, MAX_PLUGIN_PACKAGE_BYTES, "Plugin package", &mut on_progress).await?; self.guard_installer(PluginPackageInstaller::new(self.root_dir.clone(), self.app_version.clone())?) - .install_bytes_with_expectation(&bytes, policy, None, super::installer::PluginInstallSource::Url, false) + .install_bytes_with_expectation(&package, policy, None, super::installer::PluginInstallSource::Url, false) } - async fn download_limited(&self, url: Url, max_bytes: usize, label: &str) -> Result, String> { - let response = self - .client - .get(url.clone()) - .send() - .await - .map_err(|error| format!("Failed to download {label} from {url}: {error}"))?; - if !response.status().is_success() { - return Err(format!("Failed to download {label} from {url}: HTTP {}", response.status())); - } - if response.content_length().is_some_and(|length| length > max_bytes as u64) { - return Err(format!("{label} exceeds {max_bytes} bytes")); - } - let mut bytes = Vec::new(); - let mut stream = response.bytes_stream(); - while let Some(chunk) = stream.next().await { - let chunk = chunk.map_err(|error| format!("Failed to read {label} from {url}: {error}"))?; - if bytes.len().saturating_add(chunk.len()) > max_bytes { - return Err(format!("{label} exceeds {max_bytes} bytes")); - } - bytes.extend_from_slice(&chunk); - } - Ok(bytes) + /// Thin wrapper over the shared resilient download layer: retries transient + /// mid-body connection drops (the historical flaky "error decoding response + /// body" install failures) and resumes from partial bytes via Range requests. + /// Integrity remains caller-side (size + SHA-256 + signature), so a bad + /// resume can never be installed. + async fn download_limited( + &self, + url: Url, + max_bytes: usize, + label: &str, + on_progress: impl FnMut(u64, Option), + ) -> Result, String> { + dbx_platform::download::resilient_download_bytes( + &self.client, + url, + label, + max_bytes, + dbx_platform::download::RESILIENT_DOWNLOAD_ATTEMPTS, + on_progress, + ) + .await } }