mirror of
https://github.com/openai/codex.git
synced 2026-10-19 08:16:48 +00:00
Explain session creation failures during server shutdown (#52721)
## Why
Graceful shutdown rejects new sessions with an invalid-request error. Give clients a structured shutdown reason and show command center users why session creation is paused.
## What changed
- Add `data: {"reason":"serverShuttingDown"}` to app-server shutdown admission errors while retaining JSON-RPC code `-32600`.
- Show a dedicated shutdown message explaining the wait for active turns and automatic reconnection for non-embedded servers. Recognize older servers by their exact draining message only when error data is absent, and preserve retained-worktree cleanup guidance.
- Keep the command center reconnect presentation when a shutdown error covers the initial overview and no conversation is open.
## Testing
Extend admission and WebSocket shutdown assertions, add embedded and daemon shutdown message snapshots, and cover reconnection with a shutdown error over the initial command center.
GitOrigin-RevId: f167144c0ddc77872afdf3daac8bb1c89f3425b4
This commit is contained in:
@@ -19,6 +19,12 @@ These errors are not evidence that history is exhausted; callers can retry.
|
||||
A successful response with `nextCursor: null` still indicates exhaustion.
|
||||
Default scan-and-repair requests retain their filesystem fallback.
|
||||
|
||||
# Shutdown admission errors
|
||||
|
||||
When graceful shutdown rejects new work, app-server returns JSON-RPC error
|
||||
`-32600` with `data: {"reason":"serverShuttingDown"}`. Clients can use the
|
||||
reason to distinguish shutdown from other invalid requests.
|
||||
|
||||
# Thread read state (experimental)
|
||||
|
||||
Local durable ordinary user threads expose a `readState` on the
|
||||
|
||||
@@ -8,7 +8,9 @@ pub(crate) const OVERLOADED_ERROR_CODE: i64 = -32001;
|
||||
pub const INPUT_TOO_LARGE_ERROR_CODE: &str = "input_too_large";
|
||||
|
||||
pub(crate) fn server_draining_error() -> JSONRPCErrorError {
|
||||
invalid_request("Server is draining; retry after reconnecting")
|
||||
let mut error = invalid_request("Server is draining; retry after reconnecting");
|
||||
error.data = Some(serde_json::json!({ "reason": "serverShuttingDown" }));
|
||||
error
|
||||
}
|
||||
|
||||
pub(crate) fn invalid_request(message: impl Into<String>) -> JSONRPCErrorError {
|
||||
|
||||
@@ -17,7 +17,10 @@ fn drain_rejects_new_work_without_waiting_for_admitted_work() {
|
||||
|
||||
let error = admission.admit().err().expect("new request rejected");
|
||||
assert_eq!(error.code, INVALID_REQUEST_ERROR_CODE);
|
||||
assert_eq!(error.data, None);
|
||||
assert_eq!(
|
||||
error.data,
|
||||
Some(serde_json::json!({ "reason": "serverShuttingDown" }))
|
||||
);
|
||||
drop(in_flight);
|
||||
assert_eq!(*active.borrow(), 1);
|
||||
drop(automatic);
|
||||
|
||||
@@ -64,7 +64,10 @@ async fn websocket_transport_ctrl_c_waits_for_running_turn_before_exit() -> Resu
|
||||
send_turn_start_request(&mut ws, /*id*/ 4, &thread_id).await?;
|
||||
let rejected = read_error_for_id(&mut ws, /*id*/ 4).await?;
|
||||
assert_eq!(rejected.error.code, -32600);
|
||||
assert_eq!(rejected.error.data, None);
|
||||
assert_eq!(
|
||||
rejected.error.data,
|
||||
Some(json!({ "reason": "serverShuttingDown" }))
|
||||
);
|
||||
|
||||
send_request(
|
||||
&mut ws,
|
||||
|
||||
@@ -22,6 +22,14 @@ impl Renderable for AgentsOverviewErrorHeader {
|
||||
|
||||
impl App {
|
||||
pub(in crate::app) fn add_agents_overview_error(&mut self, message: String) {
|
||||
self.show_agents_overview_error(/*title*/ None, message);
|
||||
}
|
||||
|
||||
pub(in crate::app) fn show_agents_overview_error(
|
||||
&mut self,
|
||||
title: Option<&'static str>,
|
||||
message: String,
|
||||
) {
|
||||
if self
|
||||
.chat_widget
|
||||
.selected_index_for_present_view(AGENTS_OVERVIEW_VIEW_ID)
|
||||
@@ -34,13 +42,16 @@ impl App {
|
||||
}];
|
||||
self.chat_widget.show_selection_view(SelectionViewParams {
|
||||
header: Box::new(AgentsOverviewErrorHeader(vec![
|
||||
Line::from("Unable to complete action".bold()),
|
||||
Line::from(title.unwrap_or("Unable to complete action").bold()),
|
||||
Line::from(message.clone().dim()),
|
||||
])),
|
||||
items,
|
||||
..SelectionViewParams::picker()
|
||||
});
|
||||
}
|
||||
self.chat_widget.add_error_message(message);
|
||||
self.chat_widget.add_error_message(match title {
|
||||
Some(title) => format!("{title}: {message}"),
|
||||
None => message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,9 +31,10 @@ impl App {
|
||||
fn agents_overview_retained_worktree_error(
|
||||
&mut self,
|
||||
checkout: &codex_worktree::ManagedWorktree,
|
||||
title: Option<&'static str>,
|
||||
reason: impl std::fmt::Display,
|
||||
) {
|
||||
self.add_agents_overview_error(format!("{reason} A checkout was retained at {}; remove it with `git worktree remove <checkout-path>` from the source repository if it is no longer needed.", checkout.root.display()));
|
||||
self.show_agents_overview_error(title, format!("{reason} A checkout was retained at {}; remove it with `git worktree remove <checkout-path>` from the source repository if it is no longer needed.", checkout.root.display()));
|
||||
}
|
||||
|
||||
pub(in crate::app) async fn new_agents_overview_session(
|
||||
@@ -106,6 +107,7 @@ impl App {
|
||||
if let Some((_, checkout)) = &managed_worktree {
|
||||
self.agents_overview_retained_worktree_error(
|
||||
checkout,
|
||||
/*title*/ None,
|
||||
"Could not start the session.",
|
||||
);
|
||||
}
|
||||
@@ -118,6 +120,7 @@ impl App {
|
||||
if let Some((_, checkout)) = &managed_worktree {
|
||||
self.agents_overview_retained_worktree_error(
|
||||
checkout,
|
||||
/*title*/ None,
|
||||
"Could not load the new session settings.",
|
||||
);
|
||||
}
|
||||
@@ -164,13 +167,34 @@ impl App {
|
||||
let started = match result {
|
||||
Ok(started) => started,
|
||||
Err(error) => {
|
||||
if let Some((_, checkout)) = &managed_worktree {
|
||||
self.agents_overview_retained_worktree_error(
|
||||
checkout,
|
||||
format!("Failed to start session: {error}"),
|
||||
// Older servers omit the reason. Only use their specific message when
|
||||
// data is absent, so other invalid requests aren't misclassified.
|
||||
let (title, message) = if matches!(
|
||||
error.downcast_ref::<TypedRequestError>(),
|
||||
Some(TypedRequestError::Server { source, .. })
|
||||
if source.code == -32600
|
||||
&& match &source.data {
|
||||
Some(data) => data.get("reason").and_then(serde_json::Value::as_str)
|
||||
== Some("serverShuttingDown"),
|
||||
None => source.message == "Server is draining; retry after reconnecting",
|
||||
}
|
||||
) {
|
||||
let mut message = String::from(
|
||||
"New sessions are paused while active turns finish or are canceled. The default shutdown wait is up to one minute.",
|
||||
);
|
||||
if !matches!(self.app_server_target, AppServerTarget::Embedded) {
|
||||
message.push_str(
|
||||
" If the server restarts, Codex will reconnect automatically.",
|
||||
);
|
||||
}
|
||||
(Some("Codex's background server is shutting down"), message)
|
||||
} else {
|
||||
self.add_agents_overview_error(format!("Failed to start session: {error}"));
|
||||
(None, format!("Failed to start session: {error}"))
|
||||
};
|
||||
if let Some((_, checkout)) = &managed_worktree {
|
||||
self.agents_overview_retained_worktree_error(checkout, title, message);
|
||||
} else {
|
||||
self.show_agents_overview_error(title, message);
|
||||
}
|
||||
return Ok(AppRunControl::Continue);
|
||||
}
|
||||
@@ -193,7 +217,7 @@ impl App {
|
||||
};
|
||||
if let Err(error) = result {
|
||||
let _ = app_server.thread_unsubscribe(thread_id).await;
|
||||
self.agents_overview_retained_worktree_error(checkout, error);
|
||||
self.agents_overview_retained_worktree_error(checkout, /*title*/ None, error);
|
||||
return Ok(AppRunControl::Continue);
|
||||
}
|
||||
}
|
||||
@@ -216,6 +240,7 @@ impl App {
|
||||
if let Some((_, checkout)) = &managed_worktree {
|
||||
self.agents_overview_retained_worktree_error(
|
||||
checkout,
|
||||
/*title*/ None,
|
||||
"Could not open the new session.",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -228,6 +228,11 @@ impl App {
|
||||
.chat_widget
|
||||
.selected_index_for_active_view(agents_overview::AGENTS_OVERVIEW_VIEW_ID)
|
||||
.is_some()
|
||||
|| (self.current_displayed_thread_id().is_none()
|
||||
&& self
|
||||
.chat_widget
|
||||
.selected_index_for_present_view(agents_overview::AGENTS_OVERVIEW_VIEW_ID)
|
||||
.is_some())
|
||||
{
|
||||
if let Ok(mut state) = self.agents_overview.view_state.lock() {
|
||||
state.connection_notice = Some("Reconnecting — agent list is stale");
|
||||
|
||||
@@ -536,8 +536,19 @@ async fn command_center_new_read_failure_keeps_overview_and_does_not_start() ->
|
||||
for capability in [
|
||||
HistoryCapabilities::ConfigReadFails,
|
||||
HistoryCapabilities::ThreadStartFails,
|
||||
HistoryCapabilities::ThreadStartWhileShuttingDown,
|
||||
HistoryCapabilities::ThreadStartDuringLegacyShutdown,
|
||||
] {
|
||||
let (mut app, mut events, _) = make_test_app_with_channels().await;
|
||||
if capability == HistoryCapabilities::ThreadStartDuringLegacyShutdown {
|
||||
app.app_server_target = AppServerTarget::LocalDaemon {
|
||||
endpoint: crate::RemoteAppServerEndpoint::WebSocket {
|
||||
websocket_url: "ws://127.0.0.1:1".into(),
|
||||
auth_token: None,
|
||||
},
|
||||
allow_embedded_fallback: false,
|
||||
};
|
||||
}
|
||||
trust_launch_folder(&mut app);
|
||||
app.harness_overrides.model = Some("local-model".into());
|
||||
let view = app.agents_overview_view(Vec::new(), /*selected_thread_id*/ None);
|
||||
@@ -581,11 +592,29 @@ async fn command_center_new_read_failure_keeps_overview_and_does_not_start() ->
|
||||
_ => None,
|
||||
})
|
||||
.find(|message| {
|
||||
message.contains("Failed to") || message.contains("Unable to check folder trust")
|
||||
message.contains("Failed to")
|
||||
|| message.contains("Unable to check folder trust")
|
||||
|| message.contains("Codex's background server is shutting down")
|
||||
})
|
||||
.expect("visible read error");
|
||||
if capability == HistoryCapabilities::ConfigReadFails {
|
||||
insta::assert_snapshot!(error, @"■ Unable to check folder trust: config/read failed while checking remote project trust");
|
||||
} else if matches!(
|
||||
capability,
|
||||
HistoryCapabilities::ThreadStartWhileShuttingDown
|
||||
| HistoryCapabilities::ThreadStartDuringLegacyShutdown
|
||||
) {
|
||||
insta::assert_snapshot!(
|
||||
if matches!(app.app_server_target, AppServerTarget::Embedded) {
|
||||
"command_center_embedded_server_shutting_down"
|
||||
} else {
|
||||
"command_center_daemon_shutting_down"
|
||||
},
|
||||
crate::chatwidget::tests::helpers::render_bottom_popup(
|
||||
&app.chat_widget,
|
||||
/*width*/ 80,
|
||||
)
|
||||
);
|
||||
} else {
|
||||
insta::assert_snapshot!(
|
||||
"command_center_session_start_error",
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
use super::active_reconnect::drain_history;
|
||||
use super::disconnect::serve_reconnect_requests;
|
||||
use super::*;
|
||||
use crate::app::reconnect::ReconnectPresentation;
|
||||
use crate::app::reconnect::reconnect;
|
||||
use crate::app_event::AgentsOverviewThreadRefresh;
|
||||
use crate::app_event::WindowsSandboxEnableMode;
|
||||
@@ -265,6 +264,13 @@ async fn reconnect_daemon_command_center_after_socket_replacement_without_a_conv
|
||||
.is_err()
|
||||
);
|
||||
let disconnected = session.next_event().await.unwrap();
|
||||
// A shutdown error can cover the initial command center when its first New fails.
|
||||
if previous_thread.is_none() && overview_initialized {
|
||||
app.show_agents_overview_error(
|
||||
Some("Codex's background server is shutting down"),
|
||||
"Waiting for active turns.".into(),
|
||||
);
|
||||
}
|
||||
app.handle_app_server_event(&session, disconnected).await;
|
||||
assert!(app.reconnect.offline);
|
||||
assert!(app.agents_overview.last_messages.is_empty());
|
||||
@@ -321,7 +327,7 @@ async fn reconnect_daemon_command_center_after_socket_replacement_without_a_conv
|
||||
previous_thread,
|
||||
/*remote_cwd*/ None,
|
||||
session.thread_tool_transport(),
|
||||
ReconnectPresentation::Overview,
|
||||
app.reconnect.presentation,
|
||||
)
|
||||
.await?;
|
||||
app.finish_reconnect(
|
||||
|
||||
@@ -349,6 +349,8 @@ pub(super) enum HistoryCapabilities {
|
||||
ItemsAndSummaryTurnsFail,
|
||||
ThreadListFails,
|
||||
ThreadStartFails,
|
||||
ThreadStartWhileShuttingDown,
|
||||
ThreadStartDuringLegacyShutdown,
|
||||
ConfigReadUnsupported(i64),
|
||||
ConfigReadFails,
|
||||
ConfigReadUnknownVoice,
|
||||
@@ -637,15 +639,30 @@ pub(super) async fn start_recording_app_server_with_realtime_speech(
|
||||
message: "method not found".to_string(),
|
||||
},
|
||||
})
|
||||
} else if history_capabilities == HistoryCapabilities::ThreadStartFails
|
||||
&& request.method == "thread/start"
|
||||
} else if matches!(
|
||||
history_capabilities,
|
||||
HistoryCapabilities::ThreadStartFails
|
||||
| HistoryCapabilities::ThreadStartWhileShuttingDown
|
||||
| HistoryCapabilities::ThreadStartDuringLegacyShutdown
|
||||
) && request.method == "thread/start"
|
||||
{
|
||||
let (code, message, data) = match history_capabilities {
|
||||
HistoryCapabilities::ThreadStartWhileShuttingDown => (
|
||||
-32600,
|
||||
"not accepting new actions",
|
||||
Some(serde_json::json!({ "reason": "serverShuttingDown" })),
|
||||
),
|
||||
HistoryCapabilities::ThreadStartDuringLegacyShutdown => {
|
||||
(-32600, "Server is draining; retry after reconnecting", None)
|
||||
}
|
||||
_ => (-32603, "replacement unavailable", None),
|
||||
};
|
||||
JSONRPCMessage::Error(JSONRPCError {
|
||||
id: request_id,
|
||||
error: JSONRPCErrorError {
|
||||
code: -32603,
|
||||
data: None,
|
||||
message: "replacement unavailable".to_string(),
|
||||
code,
|
||||
data,
|
||||
message: message.to_string(),
|
||||
},
|
||||
})
|
||||
} else if request.method == "thread/list"
|
||||
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
---
|
||||
source: tui/src/app/tests/background_task_defaults_tests.rs
|
||||
assertion_line: 609
|
||||
expression: "crate::chatwidget::tests::helpers::render_bottom_popup(&app.chat_widget, 80,)"
|
||||
---
|
||||
Codex's background server is shutting down
|
||||
New sessions are paused while active turns finish or are canceled. The
|
||||
default shutdown wait is up to one minute. If the server restarts, Codex
|
||||
will reconnect automatically.
|
||||
|
||||
|
||||
› 1. Return to command center
|
||||
|
||||
enter select · esc back
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
---
|
||||
source: tui/src/app/tests/background_task_defaults_tests.rs
|
||||
assertion_line: 609
|
||||
expression: "crate::chatwidget::tests::helpers::render_bottom_popup(&app.chat_widget, 80,)"
|
||||
---
|
||||
Codex's background server is shutting down
|
||||
New sessions are paused while active turns finish or are canceled. The
|
||||
default shutdown wait is up to one minute.
|
||||
|
||||
|
||||
› 1. Return to command center
|
||||
|
||||
enter select · esc back
|
||||
Reference in New Issue
Block a user