Retry timed-out cloud config loads in the background (#40637)

## Why

A startup timeout leaves the cloud config loader with a cached error, while the
normal background refresh interval delays recovery for 15 minutes.

## What changed

- Retry timed-out cloud config bundle loads after 5 seconds through the existing
  background worker.
- Keep serving the cached snapshot to concurrent readers instead of triggering
  additional fetches.
- Restore the normal 15-minute refresh interval after a successful retry or a
  non-timeout error.

## Testing

Added paused-time tests covering repeated startup timeouts, successful recovery,
application of the recovered bundle to later config loads, and restoration of
the normal refresh interval after both success and validation failure.

GitOrigin-RevId: da62abbae9be7f40af61f4c4adaf4ea5c337ec88
This commit is contained in:
Abhinav authored and copyberry committed 2026-08-25 16:38:46 +00:00
1 parent 20c3f9733f
commit e44bd9b64d
2 files changed
+163 -2

No files matched your search

+13 -1
View File
@@ -34,6 +34,7 @@ use tokio::time::timeout;
pub(crate) const CLOUD_CONFIG_BUNDLE_TIMEOUT: Duration = Duration::from_secs(15);
const CLOUD_CONFIG_BUNDLE_MAX_ATTEMPTS: usize = 5;
const CLOUD_CONFIG_BUNDLE_CACHE_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60);
const CLOUD_CONFIG_BUNDLE_TIMEOUT_RETRY_INTERVAL: Duration = Duration::from_secs(5);
const CLOUD_CONFIG_BUNDLE_LOAD_FAILED_MESSAGE: &str =
"Failed to load cloud config bundle (workspace-managed policies).";
const CLOUD_CONFIG_BUNDLE_AUTH_RECOVERY_FAILED_MESSAGE: &str = concat!(
@@ -459,7 +460,18 @@ where
pub(crate) async fn refresh_cache_in_background(&self) {
loop {
sleep(CLOUD_CONFIG_BUNDLE_CACHE_REFRESH_INTERVAL).await;
let mut refresh_interval = CLOUD_CONFIG_BUNDLE_CACHE_REFRESH_INTERVAL;
if let Some(latest_bundle) = self.latest_bundle.get()
&& matches!(
&*latest_bundle.lock().await,
Err(error) if error.code() == CloudConfigBundleLoadErrorCode::Timeout
)
{
// Recover startup timeouts through this worker without making
// readers fetch concurrently or extending the startup deadline.
refresh_interval = CLOUD_CONFIG_BUNDLE_TIMEOUT_RETRY_INTERVAL;
}
sleep(refresh_interval).await;
match timeout(self.timeout, self.refresh_cache_once()).await {
Ok(true) => {}
Ok(false) => break,