Explain session creation failures during server shutdown (#52721)

## Why

Graceful shutdown rejects new sessions with an invalid-request error. Give clients a structured shutdown reason and show command center users why session creation is paused.

## What changed

- Add `data: {"reason":"serverShuttingDown"}` to app-server shutdown admission errors while retaining JSON-RPC code `-32600`.
- Show a dedicated shutdown message explaining the wait for active turns and automatic reconnection for non-embedded servers. Recognize older servers by their exact draining message only when error data is absent, and preserve retained-worktree cleanup guidance.
- Keep the command center reconnect presentation when a shutdown error covers the initial overview and no conversation is open.

## Testing

Extend admission and WebSocket shutdown assertions, add embedded and daemon shutdown message snapshots, and cover reconnection with a shutdown error over the initial command center.

GitOrigin-RevId: f167144c0ddc77872afdf3daac8bb1c89f3425b4
This commit is contained in:
Eric Traut
2026-10-09 23:16:46 +00:00
committed by copyberry
parent 4aa94dce27
commit 07757f448f
12 changed files with 154 additions and 20 deletions
+6
View File
@@ -19,6 +19,12 @@ These errors are not evidence that history is exhausted; callers can retry.
A successful response with `nextCursor: null` still indicates exhaustion.
Default scan-and-repair requests retain their filesystem fallback.
# Shutdown admission errors
When graceful shutdown rejects new work, app-server returns JSON-RPC error
`-32600` with `data: {"reason":"serverShuttingDown"}`. Clients can use the
reason to distinguish shutdown from other invalid requests.
# Thread read state (experimental)
Local durable ordinary user threads expose a `readState` on the
+3 -1
View File
@@ -8,7 +8,9 @@ pub(crate) const OVERLOADED_ERROR_CODE: i64 = -32001;
pub const INPUT_TOO_LARGE_ERROR_CODE: &str = "input_too_large";
pub(crate) fn server_draining_error() -> JSONRPCErrorError {
invalid_request("Server is draining; retry after reconnecting")
let mut error = invalid_request("Server is draining; retry after reconnecting");
error.data = Some(serde_json::json!({ "reason": "serverShuttingDown" }));
error
}
pub(crate) fn invalid_request(message: impl Into<String>) -> JSONRPCErrorError {
@@ -17,7 +17,10 @@ fn drain_rejects_new_work_without_waiting_for_admitted_work() {
let error = admission.admit().err().expect("new request rejected");
assert_eq!(error.code, INVALID_REQUEST_ERROR_CODE);
assert_eq!(error.data, None);
assert_eq!(
error.data,
Some(serde_json::json!({ "reason": "serverShuttingDown" }))
);
drop(in_flight);
assert_eq!(*active.borrow(), 1);
drop(automatic);
@@ -64,7 +64,10 @@ async fn websocket_transport_ctrl_c_waits_for_running_turn_before_exit() -> Resu
send_turn_start_request(&mut ws, /*id*/ 4, &thread_id).await?;
let rejected = read_error_for_id(&mut ws, /*id*/ 4).await?;
assert_eq!(rejected.error.code, -32600);
assert_eq!(rejected.error.data, None);
assert_eq!(
rejected.error.data,
Some(json!({ "reason": "serverShuttingDown" }))
);
send_request(
&mut ws,
+13 -2
View File
@@ -22,6 +22,14 @@ impl Renderable for AgentsOverviewErrorHeader {
impl App {
pub(in crate::app) fn add_agents_overview_error(&mut self, message: String) {
self.show_agents_overview_error(/*title*/ None, message);
}
pub(in crate::app) fn show_agents_overview_error(
&mut self,
title: Option<&'static str>,
message: String,
) {
if self
.chat_widget
.selected_index_for_present_view(AGENTS_OVERVIEW_VIEW_ID)
@@ -34,13 +42,16 @@ impl App {
}];
self.chat_widget.show_selection_view(SelectionViewParams {
header: Box::new(AgentsOverviewErrorHeader(vec![
Line::from("Unable to complete action".bold()),
Line::from(title.unwrap_or("Unable to complete action").bold()),
Line::from(message.clone().dim()),
])),
items,
..SelectionViewParams::picker()
});
}
self.chat_widget.add_error_message(message);
self.chat_widget.add_error_message(match title {
Some(title) => format!("{title}: {message}"),
None => message,
});
}
}
+32 -7
View File
@@ -31,9 +31,10 @@ impl App {
fn agents_overview_retained_worktree_error(
&mut self,
checkout: &codex_worktree::ManagedWorktree,
title: Option<&'static str>,
reason: impl std::fmt::Display,
) {
self.add_agents_overview_error(format!("{reason} A checkout was retained at {}; remove it with `git worktree remove <checkout-path>` from the source repository if it is no longer needed.", checkout.root.display()));
self.show_agents_overview_error(title, format!("{reason} A checkout was retained at {}; remove it with `git worktree remove <checkout-path>` from the source repository if it is no longer needed.", checkout.root.display()));
}
pub(in crate::app) async fn new_agents_overview_session(
@@ -106,6 +107,7 @@ impl App {
if let Some((_, checkout)) = &managed_worktree {
self.agents_overview_retained_worktree_error(
checkout,
/*title*/ None,
"Could not start the session.",
);
}
@@ -118,6 +120,7 @@ impl App {
if let Some((_, checkout)) = &managed_worktree {
self.agents_overview_retained_worktree_error(
checkout,
/*title*/ None,
"Could not load the new session settings.",
);
}
@@ -164,13 +167,34 @@ impl App {
let started = match result {
Ok(started) => started,
Err(error) => {
if let Some((_, checkout)) = &managed_worktree {
self.agents_overview_retained_worktree_error(
checkout,
format!("Failed to start session: {error}"),
// Older servers omit the reason. Only use their specific message when
// data is absent, so other invalid requests aren't misclassified.
let (title, message) = if matches!(
error.downcast_ref::<TypedRequestError>(),
Some(TypedRequestError::Server { source, .. })
if source.code == -32600
&& match &source.data {
Some(data) => data.get("reason").and_then(serde_json::Value::as_str)
== Some("serverShuttingDown"),
None => source.message == "Server is draining; retry after reconnecting",
}
) {
let mut message = String::from(
"New sessions are paused while active turns finish or are canceled. The default shutdown wait is up to one minute.",
);
if !matches!(self.app_server_target, AppServerTarget::Embedded) {
message.push_str(
" If the server restarts, Codex will reconnect automatically.",
);
}
(Some("Codex's background server is shutting down"), message)
} else {
self.add_agents_overview_error(format!("Failed to start session: {error}"));
(None, format!("Failed to start session: {error}"))
};
if let Some((_, checkout)) = &managed_worktree {
self.agents_overview_retained_worktree_error(checkout, title, message);
} else {
self.show_agents_overview_error(title, message);
}
return Ok(AppRunControl::Continue);
}
@@ -193,7 +217,7 @@ impl App {
};
if let Err(error) = result {
let _ = app_server.thread_unsubscribe(thread_id).await;
self.agents_overview_retained_worktree_error(checkout, error);
self.agents_overview_retained_worktree_error(checkout, /*title*/ None, error);
return Ok(AppRunControl::Continue);
}
}
@@ -216,6 +240,7 @@ impl App {
if let Some((_, checkout)) = &managed_worktree {
self.agents_overview_retained_worktree_error(
checkout,
/*title*/ None,
"Could not open the new session.",
);
}
+5
View File
@@ -228,6 +228,11 @@ impl App {
.chat_widget
.selected_index_for_active_view(agents_overview::AGENTS_OVERVIEW_VIEW_ID)
.is_some()
|| (self.current_displayed_thread_id().is_none()
&& self
.chat_widget
.selected_index_for_present_view(agents_overview::AGENTS_OVERVIEW_VIEW_ID)
.is_some())
{
if let Ok(mut state) = self.agents_overview.view_state.lock() {
state.connection_notice = Some("Reconnecting — agent list is stale");
@@ -536,8 +536,19 @@ async fn command_center_new_read_failure_keeps_overview_and_does_not_start() ->
for capability in [
HistoryCapabilities::ConfigReadFails,
HistoryCapabilities::ThreadStartFails,
HistoryCapabilities::ThreadStartWhileShuttingDown,
HistoryCapabilities::ThreadStartDuringLegacyShutdown,
] {
let (mut app, mut events, _) = make_test_app_with_channels().await;
if capability == HistoryCapabilities::ThreadStartDuringLegacyShutdown {
app.app_server_target = AppServerTarget::LocalDaemon {
endpoint: crate::RemoteAppServerEndpoint::WebSocket {
websocket_url: "ws://127.0.0.1:1".into(),
auth_token: None,
},
allow_embedded_fallback: false,
};
}
trust_launch_folder(&mut app);
app.harness_overrides.model = Some("local-model".into());
let view = app.agents_overview_view(Vec::new(), /*selected_thread_id*/ None);
@@ -581,11 +592,29 @@ async fn command_center_new_read_failure_keeps_overview_and_does_not_start() ->
_ => None,
})
.find(|message| {
message.contains("Failed to") || message.contains("Unable to check folder trust")
message.contains("Failed to")
|| message.contains("Unable to check folder trust")
|| message.contains("Codex's background server is shutting down")
})
.expect("visible read error");
if capability == HistoryCapabilities::ConfigReadFails {
insta::assert_snapshot!(error, @"■ Unable to check folder trust: config/read failed while checking remote project trust");
} else if matches!(
capability,
HistoryCapabilities::ThreadStartWhileShuttingDown
| HistoryCapabilities::ThreadStartDuringLegacyShutdown
) {
insta::assert_snapshot!(
if matches!(app.app_server_target, AppServerTarget::Embedded) {
"command_center_embedded_server_shutting_down"
} else {
"command_center_daemon_shutting_down"
},
crate::chatwidget::tests::helpers::render_bottom_popup(
&app.chat_widget,
/*width*/ 80,
)
);
} else {
insta::assert_snapshot!(
"command_center_session_start_error",
@@ -3,7 +3,6 @@
use super::active_reconnect::drain_history;
use super::disconnect::serve_reconnect_requests;
use super::*;
use crate::app::reconnect::ReconnectPresentation;
use crate::app::reconnect::reconnect;
use crate::app_event::AgentsOverviewThreadRefresh;
use crate::app_event::WindowsSandboxEnableMode;
@@ -265,6 +264,13 @@ async fn reconnect_daemon_command_center_after_socket_replacement_without_a_conv
.is_err()
);
let disconnected = session.next_event().await.unwrap();
// A shutdown error can cover the initial command center when its first New fails.
if previous_thread.is_none() && overview_initialized {
app.show_agents_overview_error(
Some("Codex's background server is shutting down"),
"Waiting for active turns.".into(),
);
}
app.handle_app_server_event(&session, disconnected).await;
assert!(app.reconnect.offline);
assert!(app.agents_overview.last_messages.is_empty());
@@ -321,7 +327,7 @@ async fn reconnect_daemon_command_center_after_socket_replacement_without_a_conv
previous_thread,
/*remote_cwd*/ None,
session.thread_tool_transport(),
ReconnectPresentation::Overview,
app.reconnect.presentation,
)
.await?;
app.finish_reconnect(
@@ -349,6 +349,8 @@ pub(super) enum HistoryCapabilities {
ItemsAndSummaryTurnsFail,
ThreadListFails,
ThreadStartFails,
ThreadStartWhileShuttingDown,
ThreadStartDuringLegacyShutdown,
ConfigReadUnsupported(i64),
ConfigReadFails,
ConfigReadUnknownVoice,
@@ -637,15 +639,30 @@ pub(super) async fn start_recording_app_server_with_realtime_speech(
message: "method not found".to_string(),
},
})
} else if history_capabilities == HistoryCapabilities::ThreadStartFails
&& request.method == "thread/start"
} else if matches!(
history_capabilities,
HistoryCapabilities::ThreadStartFails
| HistoryCapabilities::ThreadStartWhileShuttingDown
| HistoryCapabilities::ThreadStartDuringLegacyShutdown
) && request.method == "thread/start"
{
let (code, message, data) = match history_capabilities {
HistoryCapabilities::ThreadStartWhileShuttingDown => (
-32600,
"not accepting new actions",
Some(serde_json::json!({ "reason": "serverShuttingDown" })),
),
HistoryCapabilities::ThreadStartDuringLegacyShutdown => {
(-32600, "Server is draining; retry after reconnecting", None)
}
_ => (-32603, "replacement unavailable", None),
};
JSONRPCMessage::Error(JSONRPCError {
id: request_id,
error: JSONRPCErrorError {
code: -32603,
data: None,
message: "replacement unavailable".to_string(),
code,
data,
message: message.to_string(),
},
})
} else if request.method == "thread/list"
@@ -0,0 +1,14 @@
---
source: tui/src/app/tests/background_task_defaults_tests.rs
assertion_line: 609
expression: "crate::chatwidget::tests::helpers::render_bottom_popup(&app.chat_widget, 80,)"
---
Codex's background server is shutting down
New sessions are paused while active turns finish or are canceled. The
default shutdown wait is up to one minute. If the server restarts, Codex
will reconnect automatically.
› 1. Return to command center
enter select · esc back
@@ -0,0 +1,13 @@
---
source: tui/src/app/tests/background_task_defaults_tests.rs
assertion_line: 609
expression: "crate::chatwidget::tests::helpers::render_bottom_popup(&app.chat_widget, 80,)"
---
Codex's background server is shutting down
New sessions are paused while active turns finish or are canceled. The
default shutdown wait is up to one minute.
› 1. Return to command center
enter select · esc back