mirror of
https://github.com/openai/codex.git
synced 2026-11-13 14:46:14 +00:00
app-server: keep the model cache warm (#28699)
## Why The app server is long-lived, but its shared model cache otherwise refreshes only when a caller needs it. Once the five-minute cache expires, starting a thread or calling `model/list` can wait for `/models` on the request path. Refresh the cache in the background before it expires so foreground callers normally use fresh local state. ## What changed - Start an app-server worker that refreshes models immediately and then every three minutes using the existing models-manager API. - Hold only a weak reference to the models manager between refreshes, so the worker does not extend its lifetime. - Stop scheduling refreshes when the app-server lifecycle handle is shut down or dropped. A refresh already in progress is allowed to finish. - Adjust affected app-server test fixtures to distinguish the background `/models` probe from the connection they are testing. The existing models-manager cache, refresh strategies, auth handling, ETag behavior, and concurrency semantics are unchanged. ## Testing - `models_refresh_worker::tests::refreshes_immediately_periodically_and_stops_when_dropped` - `suite::v2::remote_control::listen_off_honors_persisted_remote_control_enable` - `suite::v2::attestation::attestation_generate_round_trip_adds_header_to_responses_websocket_handshake`
This commit is contained in:
7 files changed
+238
-58
No files matched your search
@@ -0,0 +1,65 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use codex_models_manager::manager::RefreshStrategy;
|
||||
use codex_models_manager::manager::SharedModelsManager;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
const MODELS_REFRESH_INTERVAL: Duration = Duration::from_secs(3 * 60);
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct ModelsRefreshWorker {
|
||||
shutdown: CancellationToken,
|
||||
_task: JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl ModelsRefreshWorker {
|
||||
pub(crate) fn shutdown(&self) {
|
||||
self.shutdown.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ModelsRefreshWorker {
|
||||
fn drop(&mut self) {
|
||||
self.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn spawn(models_manager: &SharedModelsManager) -> ModelsRefreshWorker {
|
||||
spawn_with_interval(models_manager, MODELS_REFRESH_INTERVAL)
|
||||
}
|
||||
|
||||
fn spawn_with_interval(
|
||||
models_manager: &SharedModelsManager,
|
||||
refresh_interval: Duration,
|
||||
) -> ModelsRefreshWorker {
|
||||
let models_manager = Arc::downgrade(models_manager);
|
||||
let shutdown = CancellationToken::new();
|
||||
let worker_shutdown = shutdown.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
loop {
|
||||
if worker_shutdown.is_cancelled() {
|
||||
break;
|
||||
}
|
||||
let Some(models_manager) = models_manager.upgrade() else {
|
||||
break;
|
||||
};
|
||||
models_manager.list_models(RefreshStrategy::Online).await;
|
||||
drop(models_manager);
|
||||
|
||||
tokio::select! {
|
||||
_ = worker_shutdown.cancelled() => break,
|
||||
_ = tokio::time::sleep(refresh_interval) => {}
|
||||
}
|
||||
}
|
||||
});
|
||||
ModelsRefreshWorker {
|
||||
shutdown,
|
||||
_task: task,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "models_refresh_worker_tests.rs"]
|
||||
mod tests;
|
||||
Reference in new issue
Block a user