mirror of
https://github.com/openai/codex.git
synced 2026-10-19 16:17:49 +00:00
Explain session creation failures during server shutdown (#52721)
## Why
Graceful shutdown rejects new sessions with an invalid-request error. Give clients a structured shutdown reason and show command center users why session creation is paused.
## What changed
- Add `data: {"reason":"serverShuttingDown"}` to app-server shutdown admission errors while retaining JSON-RPC code `-32600`.
- Show a dedicated shutdown message explaining the wait for active turns and automatic reconnection for non-embedded servers. Recognize older servers by their exact draining message only when error data is absent, and preserve retained-worktree cleanup guidance.
- Keep the command center reconnect presentation when a shutdown error covers the initial overview and no conversation is open.
## Testing
Extend admission and WebSocket shutdown assertions, add embedded and daemon shutdown message snapshots, and cover reconnection with a shutdown error over the initial command center.
GitOrigin-RevId: f167144c0ddc77872afdf3daac8bb1c89f3425b4
This commit is contained in:
@@ -19,6 +19,12 @@ These errors are not evidence that history is exhausted; callers can retry.
|
|||||||
A successful response with `nextCursor: null` still indicates exhaustion.
|
A successful response with `nextCursor: null` still indicates exhaustion.
|
||||||
Default scan-and-repair requests retain their filesystem fallback.
|
Default scan-and-repair requests retain their filesystem fallback.
|
||||||
|
|
||||||
|
# Shutdown admission errors
|
||||||
|
|
||||||
|
When graceful shutdown rejects new work, app-server returns JSON-RPC error
|
||||||
|
`-32600` with `data: {"reason":"serverShuttingDown"}`. Clients can use the
|
||||||
|
reason to distinguish shutdown from other invalid requests.
|
||||||
|
|
||||||
# Thread read state (experimental)
|
# Thread read state (experimental)
|
||||||
|
|
||||||
Local durable ordinary user threads expose a `readState` on the
|
Local durable ordinary user threads expose a `readState` on the
|
||||||
|
|||||||
@@ -8,7 +8,9 @@ pub(crate) const OVERLOADED_ERROR_CODE: i64 = -32001;
|
|||||||
pub const INPUT_TOO_LARGE_ERROR_CODE: &str = "input_too_large";
|
pub const INPUT_TOO_LARGE_ERROR_CODE: &str = "input_too_large";
|
||||||
|
|
||||||
pub(crate) fn server_draining_error() -> JSONRPCErrorError {
|
pub(crate) fn server_draining_error() -> JSONRPCErrorError {
|
||||||
invalid_request("Server is draining; retry after reconnecting")
|
let mut error = invalid_request("Server is draining; retry after reconnecting");
|
||||||
|
error.data = Some(serde_json::json!({ "reason": "serverShuttingDown" }));
|
||||||
|
error
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn invalid_request(message: impl Into<String>) -> JSONRPCErrorError {
|
pub(crate) fn invalid_request(message: impl Into<String>) -> JSONRPCErrorError {
|
||||||
|
|||||||
@@ -17,7 +17,10 @@ fn drain_rejects_new_work_without_waiting_for_admitted_work() {
|
|||||||
|
|
||||||
let error = admission.admit().err().expect("new request rejected");
|
let error = admission.admit().err().expect("new request rejected");
|
||||||
assert_eq!(error.code, INVALID_REQUEST_ERROR_CODE);
|
assert_eq!(error.code, INVALID_REQUEST_ERROR_CODE);
|
||||||
assert_eq!(error.data, None);
|
assert_eq!(
|
||||||
|
error.data,
|
||||||
|
Some(serde_json::json!({ "reason": "serverShuttingDown" }))
|
||||||
|
);
|
||||||
drop(in_flight);
|
drop(in_flight);
|
||||||
assert_eq!(*active.borrow(), 1);
|
assert_eq!(*active.borrow(), 1);
|
||||||
drop(automatic);
|
drop(automatic);
|
||||||
|
|||||||
@@ -64,7 +64,10 @@ async fn websocket_transport_ctrl_c_waits_for_running_turn_before_exit() -> Resu
|
|||||||
send_turn_start_request(&mut ws, /*id*/ 4, &thread_id).await?;
|
send_turn_start_request(&mut ws, /*id*/ 4, &thread_id).await?;
|
||||||
let rejected = read_error_for_id(&mut ws, /*id*/ 4).await?;
|
let rejected = read_error_for_id(&mut ws, /*id*/ 4).await?;
|
||||||
assert_eq!(rejected.error.code, -32600);
|
assert_eq!(rejected.error.code, -32600);
|
||||||
assert_eq!(rejected.error.data, None);
|
assert_eq!(
|
||||||
|
rejected.error.data,
|
||||||
|
Some(json!({ "reason": "serverShuttingDown" }))
|
||||||
|
);
|
||||||
|
|
||||||
send_request(
|
send_request(
|
||||||
&mut ws,
|
&mut ws,
|
||||||
|
|||||||
@@ -22,6 +22,14 @@ impl Renderable for AgentsOverviewErrorHeader {
|
|||||||
|
|
||||||
impl App {
|
impl App {
|
||||||
pub(in crate::app) fn add_agents_overview_error(&mut self, message: String) {
|
pub(in crate::app) fn add_agents_overview_error(&mut self, message: String) {
|
||||||
|
self.show_agents_overview_error(/*title*/ None, message);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(in crate::app) fn show_agents_overview_error(
|
||||||
|
&mut self,
|
||||||
|
title: Option<&'static str>,
|
||||||
|
message: String,
|
||||||
|
) {
|
||||||
if self
|
if self
|
||||||
.chat_widget
|
.chat_widget
|
||||||
.selected_index_for_present_view(AGENTS_OVERVIEW_VIEW_ID)
|
.selected_index_for_present_view(AGENTS_OVERVIEW_VIEW_ID)
|
||||||
@@ -34,13 +42,16 @@ impl App {
|
|||||||
}];
|
}];
|
||||||
self.chat_widget.show_selection_view(SelectionViewParams {
|
self.chat_widget.show_selection_view(SelectionViewParams {
|
||||||
header: Box::new(AgentsOverviewErrorHeader(vec![
|
header: Box::new(AgentsOverviewErrorHeader(vec![
|
||||||
Line::from("Unable to complete action".bold()),
|
Line::from(title.unwrap_or("Unable to complete action").bold()),
|
||||||
Line::from(message.clone().dim()),
|
Line::from(message.clone().dim()),
|
||||||
])),
|
])),
|
||||||
items,
|
items,
|
||||||
..SelectionViewParams::picker()
|
..SelectionViewParams::picker()
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
self.chat_widget.add_error_message(message);
|
self.chat_widget.add_error_message(match title {
|
||||||
|
Some(title) => format!("{title}: {message}"),
|
||||||
|
None => message,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,9 +31,10 @@ impl App {
|
|||||||
fn agents_overview_retained_worktree_error(
|
fn agents_overview_retained_worktree_error(
|
||||||
&mut self,
|
&mut self,
|
||||||
checkout: &codex_worktree::ManagedWorktree,
|
checkout: &codex_worktree::ManagedWorktree,
|
||||||
|
title: Option<&'static str>,
|
||||||
reason: impl std::fmt::Display,
|
reason: impl std::fmt::Display,
|
||||||
) {
|
) {
|
||||||
self.add_agents_overview_error(format!("{reason} A checkout was retained at {}; remove it with `git worktree remove <checkout-path>` from the source repository if it is no longer needed.", checkout.root.display()));
|
self.show_agents_overview_error(title, format!("{reason} A checkout was retained at {}; remove it with `git worktree remove <checkout-path>` from the source repository if it is no longer needed.", checkout.root.display()));
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(in crate::app) async fn new_agents_overview_session(
|
pub(in crate::app) async fn new_agents_overview_session(
|
||||||
@@ -106,6 +107,7 @@ impl App {
|
|||||||
if let Some((_, checkout)) = &managed_worktree {
|
if let Some((_, checkout)) = &managed_worktree {
|
||||||
self.agents_overview_retained_worktree_error(
|
self.agents_overview_retained_worktree_error(
|
||||||
checkout,
|
checkout,
|
||||||
|
/*title*/ None,
|
||||||
"Could not start the session.",
|
"Could not start the session.",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -118,6 +120,7 @@ impl App {
|
|||||||
if let Some((_, checkout)) = &managed_worktree {
|
if let Some((_, checkout)) = &managed_worktree {
|
||||||
self.agents_overview_retained_worktree_error(
|
self.agents_overview_retained_worktree_error(
|
||||||
checkout,
|
checkout,
|
||||||
|
/*title*/ None,
|
||||||
"Could not load the new session settings.",
|
"Could not load the new session settings.",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -164,13 +167,34 @@ impl App {
|
|||||||
let started = match result {
|
let started = match result {
|
||||||
Ok(started) => started,
|
Ok(started) => started,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
if let Some((_, checkout)) = &managed_worktree {
|
// Older servers omit the reason. Only use their specific message when
|
||||||
self.agents_overview_retained_worktree_error(
|
// data is absent, so other invalid requests aren't misclassified.
|
||||||
checkout,
|
let (title, message) = if matches!(
|
||||||
format!("Failed to start session: {error}"),
|
error.downcast_ref::<TypedRequestError>(),
|
||||||
|
Some(TypedRequestError::Server { source, .. })
|
||||||
|
if source.code == -32600
|
||||||
|
&& match &source.data {
|
||||||
|
Some(data) => data.get("reason").and_then(serde_json::Value::as_str)
|
||||||
|
== Some("serverShuttingDown"),
|
||||||
|
None => source.message == "Server is draining; retry after reconnecting",
|
||||||
|
}
|
||||||
|
) {
|
||||||
|
let mut message = String::from(
|
||||||
|
"New sessions are paused while active turns finish or are canceled. The default shutdown wait is up to one minute.",
|
||||||
);
|
);
|
||||||
|
if !matches!(self.app_server_target, AppServerTarget::Embedded) {
|
||||||
|
message.push_str(
|
||||||
|
" If the server restarts, Codex will reconnect automatically.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
(Some("Codex's background server is shutting down"), message)
|
||||||
} else {
|
} else {
|
||||||
self.add_agents_overview_error(format!("Failed to start session: {error}"));
|
(None, format!("Failed to start session: {error}"))
|
||||||
|
};
|
||||||
|
if let Some((_, checkout)) = &managed_worktree {
|
||||||
|
self.agents_overview_retained_worktree_error(checkout, title, message);
|
||||||
|
} else {
|
||||||
|
self.show_agents_overview_error(title, message);
|
||||||
}
|
}
|
||||||
return Ok(AppRunControl::Continue);
|
return Ok(AppRunControl::Continue);
|
||||||
}
|
}
|
||||||
@@ -193,7 +217,7 @@ impl App {
|
|||||||
};
|
};
|
||||||
if let Err(error) = result {
|
if let Err(error) = result {
|
||||||
let _ = app_server.thread_unsubscribe(thread_id).await;
|
let _ = app_server.thread_unsubscribe(thread_id).await;
|
||||||
self.agents_overview_retained_worktree_error(checkout, error);
|
self.agents_overview_retained_worktree_error(checkout, /*title*/ None, error);
|
||||||
return Ok(AppRunControl::Continue);
|
return Ok(AppRunControl::Continue);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -216,6 +240,7 @@ impl App {
|
|||||||
if let Some((_, checkout)) = &managed_worktree {
|
if let Some((_, checkout)) = &managed_worktree {
|
||||||
self.agents_overview_retained_worktree_error(
|
self.agents_overview_retained_worktree_error(
|
||||||
checkout,
|
checkout,
|
||||||
|
/*title*/ None,
|
||||||
"Could not open the new session.",
|
"Could not open the new session.",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -228,6 +228,11 @@ impl App {
|
|||||||
.chat_widget
|
.chat_widget
|
||||||
.selected_index_for_active_view(agents_overview::AGENTS_OVERVIEW_VIEW_ID)
|
.selected_index_for_active_view(agents_overview::AGENTS_OVERVIEW_VIEW_ID)
|
||||||
.is_some()
|
.is_some()
|
||||||
|
|| (self.current_displayed_thread_id().is_none()
|
||||||
|
&& self
|
||||||
|
.chat_widget
|
||||||
|
.selected_index_for_present_view(agents_overview::AGENTS_OVERVIEW_VIEW_ID)
|
||||||
|
.is_some())
|
||||||
{
|
{
|
||||||
if let Ok(mut state) = self.agents_overview.view_state.lock() {
|
if let Ok(mut state) = self.agents_overview.view_state.lock() {
|
||||||
state.connection_notice = Some("Reconnecting — agent list is stale");
|
state.connection_notice = Some("Reconnecting — agent list is stale");
|
||||||
|
|||||||
@@ -536,8 +536,19 @@ async fn command_center_new_read_failure_keeps_overview_and_does_not_start() ->
|
|||||||
for capability in [
|
for capability in [
|
||||||
HistoryCapabilities::ConfigReadFails,
|
HistoryCapabilities::ConfigReadFails,
|
||||||
HistoryCapabilities::ThreadStartFails,
|
HistoryCapabilities::ThreadStartFails,
|
||||||
|
HistoryCapabilities::ThreadStartWhileShuttingDown,
|
||||||
|
HistoryCapabilities::ThreadStartDuringLegacyShutdown,
|
||||||
] {
|
] {
|
||||||
let (mut app, mut events, _) = make_test_app_with_channels().await;
|
let (mut app, mut events, _) = make_test_app_with_channels().await;
|
||||||
|
if capability == HistoryCapabilities::ThreadStartDuringLegacyShutdown {
|
||||||
|
app.app_server_target = AppServerTarget::LocalDaemon {
|
||||||
|
endpoint: crate::RemoteAppServerEndpoint::WebSocket {
|
||||||
|
websocket_url: "ws://127.0.0.1:1".into(),
|
||||||
|
auth_token: None,
|
||||||
|
},
|
||||||
|
allow_embedded_fallback: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
trust_launch_folder(&mut app);
|
trust_launch_folder(&mut app);
|
||||||
app.harness_overrides.model = Some("local-model".into());
|
app.harness_overrides.model = Some("local-model".into());
|
||||||
let view = app.agents_overview_view(Vec::new(), /*selected_thread_id*/ None);
|
let view = app.agents_overview_view(Vec::new(), /*selected_thread_id*/ None);
|
||||||
@@ -581,11 +592,29 @@ async fn command_center_new_read_failure_keeps_overview_and_does_not_start() ->
|
|||||||
_ => None,
|
_ => None,
|
||||||
})
|
})
|
||||||
.find(|message| {
|
.find(|message| {
|
||||||
message.contains("Failed to") || message.contains("Unable to check folder trust")
|
message.contains("Failed to")
|
||||||
|
|| message.contains("Unable to check folder trust")
|
||||||
|
|| message.contains("Codex's background server is shutting down")
|
||||||
})
|
})
|
||||||
.expect("visible read error");
|
.expect("visible read error");
|
||||||
if capability == HistoryCapabilities::ConfigReadFails {
|
if capability == HistoryCapabilities::ConfigReadFails {
|
||||||
insta::assert_snapshot!(error, @"■ Unable to check folder trust: config/read failed while checking remote project trust");
|
insta::assert_snapshot!(error, @"■ Unable to check folder trust: config/read failed while checking remote project trust");
|
||||||
|
} else if matches!(
|
||||||
|
capability,
|
||||||
|
HistoryCapabilities::ThreadStartWhileShuttingDown
|
||||||
|
| HistoryCapabilities::ThreadStartDuringLegacyShutdown
|
||||||
|
) {
|
||||||
|
insta::assert_snapshot!(
|
||||||
|
if matches!(app.app_server_target, AppServerTarget::Embedded) {
|
||||||
|
"command_center_embedded_server_shutting_down"
|
||||||
|
} else {
|
||||||
|
"command_center_daemon_shutting_down"
|
||||||
|
},
|
||||||
|
crate::chatwidget::tests::helpers::render_bottom_popup(
|
||||||
|
&app.chat_widget,
|
||||||
|
/*width*/ 80,
|
||||||
|
)
|
||||||
|
);
|
||||||
} else {
|
} else {
|
||||||
insta::assert_snapshot!(
|
insta::assert_snapshot!(
|
||||||
"command_center_session_start_error",
|
"command_center_session_start_error",
|
||||||
|
|||||||
@@ -3,7 +3,6 @@
|
|||||||
use super::active_reconnect::drain_history;
|
use super::active_reconnect::drain_history;
|
||||||
use super::disconnect::serve_reconnect_requests;
|
use super::disconnect::serve_reconnect_requests;
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::app::reconnect::ReconnectPresentation;
|
|
||||||
use crate::app::reconnect::reconnect;
|
use crate::app::reconnect::reconnect;
|
||||||
use crate::app_event::AgentsOverviewThreadRefresh;
|
use crate::app_event::AgentsOverviewThreadRefresh;
|
||||||
use crate::app_event::WindowsSandboxEnableMode;
|
use crate::app_event::WindowsSandboxEnableMode;
|
||||||
@@ -265,6 +264,13 @@ async fn reconnect_daemon_command_center_after_socket_replacement_without_a_conv
|
|||||||
.is_err()
|
.is_err()
|
||||||
);
|
);
|
||||||
let disconnected = session.next_event().await.unwrap();
|
let disconnected = session.next_event().await.unwrap();
|
||||||
|
// A shutdown error can cover the initial command center when its first New fails.
|
||||||
|
if previous_thread.is_none() && overview_initialized {
|
||||||
|
app.show_agents_overview_error(
|
||||||
|
Some("Codex's background server is shutting down"),
|
||||||
|
"Waiting for active turns.".into(),
|
||||||
|
);
|
||||||
|
}
|
||||||
app.handle_app_server_event(&session, disconnected).await;
|
app.handle_app_server_event(&session, disconnected).await;
|
||||||
assert!(app.reconnect.offline);
|
assert!(app.reconnect.offline);
|
||||||
assert!(app.agents_overview.last_messages.is_empty());
|
assert!(app.agents_overview.last_messages.is_empty());
|
||||||
@@ -321,7 +327,7 @@ async fn reconnect_daemon_command_center_after_socket_replacement_without_a_conv
|
|||||||
previous_thread,
|
previous_thread,
|
||||||
/*remote_cwd*/ None,
|
/*remote_cwd*/ None,
|
||||||
session.thread_tool_transport(),
|
session.thread_tool_transport(),
|
||||||
ReconnectPresentation::Overview,
|
app.reconnect.presentation,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
app.finish_reconnect(
|
app.finish_reconnect(
|
||||||
|
|||||||
@@ -349,6 +349,8 @@ pub(super) enum HistoryCapabilities {
|
|||||||
ItemsAndSummaryTurnsFail,
|
ItemsAndSummaryTurnsFail,
|
||||||
ThreadListFails,
|
ThreadListFails,
|
||||||
ThreadStartFails,
|
ThreadStartFails,
|
||||||
|
ThreadStartWhileShuttingDown,
|
||||||
|
ThreadStartDuringLegacyShutdown,
|
||||||
ConfigReadUnsupported(i64),
|
ConfigReadUnsupported(i64),
|
||||||
ConfigReadFails,
|
ConfigReadFails,
|
||||||
ConfigReadUnknownVoice,
|
ConfigReadUnknownVoice,
|
||||||
@@ -637,15 +639,30 @@ pub(super) async fn start_recording_app_server_with_realtime_speech(
|
|||||||
message: "method not found".to_string(),
|
message: "method not found".to_string(),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
} else if history_capabilities == HistoryCapabilities::ThreadStartFails
|
} else if matches!(
|
||||||
&& request.method == "thread/start"
|
history_capabilities,
|
||||||
|
HistoryCapabilities::ThreadStartFails
|
||||||
|
| HistoryCapabilities::ThreadStartWhileShuttingDown
|
||||||
|
| HistoryCapabilities::ThreadStartDuringLegacyShutdown
|
||||||
|
) && request.method == "thread/start"
|
||||||
{
|
{
|
||||||
|
let (code, message, data) = match history_capabilities {
|
||||||
|
HistoryCapabilities::ThreadStartWhileShuttingDown => (
|
||||||
|
-32600,
|
||||||
|
"not accepting new actions",
|
||||||
|
Some(serde_json::json!({ "reason": "serverShuttingDown" })),
|
||||||
|
),
|
||||||
|
HistoryCapabilities::ThreadStartDuringLegacyShutdown => {
|
||||||
|
(-32600, "Server is draining; retry after reconnecting", None)
|
||||||
|
}
|
||||||
|
_ => (-32603, "replacement unavailable", None),
|
||||||
|
};
|
||||||
JSONRPCMessage::Error(JSONRPCError {
|
JSONRPCMessage::Error(JSONRPCError {
|
||||||
id: request_id,
|
id: request_id,
|
||||||
error: JSONRPCErrorError {
|
error: JSONRPCErrorError {
|
||||||
code: -32603,
|
code,
|
||||||
data: None,
|
data,
|
||||||
message: "replacement unavailable".to_string(),
|
message: message.to_string(),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
} else if request.method == "thread/list"
|
} else if request.method == "thread/list"
|
||||||
|
|||||||
+14
@@ -0,0 +1,14 @@
|
|||||||
|
---
|
||||||
|
source: tui/src/app/tests/background_task_defaults_tests.rs
|
||||||
|
assertion_line: 609
|
||||||
|
expression: "crate::chatwidget::tests::helpers::render_bottom_popup(&app.chat_widget, 80,)"
|
||||||
|
---
|
||||||
|
Codex's background server is shutting down
|
||||||
|
New sessions are paused while active turns finish or are canceled. The
|
||||||
|
default shutdown wait is up to one minute. If the server restarts, Codex
|
||||||
|
will reconnect automatically.
|
||||||
|
|
||||||
|
|
||||||
|
› 1. Return to command center
|
||||||
|
|
||||||
|
enter select · esc back
|
||||||
+13
@@ -0,0 +1,13 @@
|
|||||||
|
---
|
||||||
|
source: tui/src/app/tests/background_task_defaults_tests.rs
|
||||||
|
assertion_line: 609
|
||||||
|
expression: "crate::chatwidget::tests::helpers::render_bottom_popup(&app.chat_widget, 80,)"
|
||||||
|
---
|
||||||
|
Codex's background server is shutting down
|
||||||
|
New sessions are paused while active turns finish or are canceled. The
|
||||||
|
default shutdown wait is up to one minute.
|
||||||
|
|
||||||
|
|
||||||
|
› 1. Return to command center
|
||||||
|
|
||||||
|
enter select · esc back
|
||||||
Reference in New Issue
Block a user