mirror of
https://github.com/openai/codex.git
synced 2026-11-18 23:02:23 +00:00
Reconnect TUI app-server sessions automatically (#41916)
## What changed - Reconnect out-of-process TUI sessions after the app-server transport drops, bootstrap a fresh client, and resume the active thread. - Preserve cached transcripts, drafts, runtime settings, task-tool capabilities, and live notification routing across the new connection. - Keep recovered or possibly submitted input paused for manual review instead of resending it, and leave conversations that cannot be resumed available as read-only cached views. - Show reconnect progress and actionable failure states while still allowing drafts to be edited or copied. ## Testing - Add app-level coverage for successful recovery, failed and unavailable resumes, input quarantine, and preserved session state. - Extend the terminal integration test to verify that a draft survives reconnection, the thread is resumed without starting a turn, and notifications arrive through the replacement connection. GitOrigin-RevId: 5dcb867c89e226a0c545d6313a77424c233cd47a
This commit is contained in:
1 parent
9a4b78579a
commit
907c34e867
20 files changed
+1082
-46
No files matched your search
@@ -50,7 +50,7 @@ pub(crate) struct AgentNavigationState {
|
|||||||
/// Spawned child threads whose instructions are owned by their parent agent.
|
/// Spawned child threads whose instructions are owned by their parent agent.
|
||||||
parent_owned_threads: HashSet<ThreadId>,
|
parent_owned_threads: HashSet<ThreadId>,
|
||||||
/// Coalesces root refreshes while rejecting replies from a previous session.
|
/// Coalesces root refreshes while rejecting replies from a previous session.
|
||||||
picker_refresh: Option<(ThreadId, Uuid)>,
|
pub(super) picker_refresh: Option<(ThreadId, Uuid)>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Direction of keyboard traversal through the stable picker order.
|
/// Direction of keyboard traversal through the stable picker order.
|
||||||
|
|||||||
@@ -370,6 +370,17 @@ impl App {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if !self.chat_widget.has_active_view()
|
||||||
|
&& self
|
||||||
|
.current_displayed_thread_id()
|
||||||
|
.is_some_and(|id| self.thread_unavailable(id))
|
||||||
|
&& !(key_event.modifiers.contains(KeyModifiers::CONTROL)
|
||||||
|
&& matches!(key_event.code, KeyCode::Char('c' | 'd')))
|
||||||
|
{
|
||||||
|
self.chat_widget.handle_disconnected_key(key_event);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if matches!(key_event.code, KeyCode::Esc)
|
if matches!(key_event.code, KeyCode::Esc)
|
||||||
&& matches!(key_event.kind, KeyEventKind::Press | KeyEventKind::Repeat)
|
&& matches!(key_event.kind, KeyEventKind::Press | KeyEventKind::Repeat)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,7 +1,11 @@
|
|||||||
//! Preserve local work after an external connection is lost. Offline input and stale
|
//! TUI-only out-of-process reconnection. Each attempt initializes a fresh client and rejoins existing
|
||||||
//! async completions are quarantined; recovery requires relaunching the TUI.
|
//! threads using ordinary resume/history semantics; no user operation is retried.
|
||||||
|
//! Offline input and old async completions are quarantined. Cached conversations stay read-only
|
||||||
|
//! until they can be reattached; command-center recovery is handled separately.
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use crate::app_server_session::ResumeModelSettings;
|
||||||
|
use crate::dynamic_tools_mcp::ThreadToolTransport;
|
||||||
|
|
||||||
#[derive(Clone, Copy, Default, PartialEq, Eq)]
|
#[derive(Clone, Copy, Default, PartialEq, Eq)]
|
||||||
pub(super) enum ReconnectPresentation {
|
pub(super) enum ReconnectPresentation {
|
||||||
@@ -13,10 +17,110 @@ pub(super) enum ReconnectPresentation {
|
|||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
pub(super) struct ReconnectState {
|
pub(super) struct ReconnectState {
|
||||||
pub(super) offline: bool,
|
pub(super) offline: bool,
|
||||||
|
pub(super) failed: bool,
|
||||||
pub(super) presentation: ReconnectPresentation,
|
pub(super) presentation: ReconnectPresentation,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) struct Reconnected {
|
||||||
|
session: AppServerSession,
|
||||||
|
bootstrap: AppServerBootstrap,
|
||||||
|
thread: Option<AppServerStartedThread>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn reconnect(
|
||||||
|
target: AppServerTarget,
|
||||||
|
config: Config,
|
||||||
|
thread_id: Option<ThreadId>,
|
||||||
|
remote_cwd: Option<PathBuf>,
|
||||||
|
task_tools: ThreadToolTransport,
|
||||||
|
presentation: ReconnectPresentation,
|
||||||
|
) -> Result<Reconnected> {
|
||||||
|
let mode = target.thread_params_mode();
|
||||||
|
let endpoint = match target {
|
||||||
|
AppServerTarget::Remote { endpoint } | AppServerTarget::LocalDaemon { endpoint } => {
|
||||||
|
endpoint
|
||||||
|
}
|
||||||
|
AppServerTarget::Embedded => {
|
||||||
|
color_eyre::eyre::bail!("in-process sessions have no connection to restore")
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if let ThreadToolTransport::Mcp(server) = &task_tools {
|
||||||
|
server.suspend();
|
||||||
|
}
|
||||||
|
if presentation == ReconnectPresentation::Conversation && thread_id.is_none() {
|
||||||
|
color_eyre::eyre::bail!(
|
||||||
|
"The initial thread may have been created, but its ID was not received. Nothing was retried. Your prompt is editable; inspect your tasks before relaunching."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Connecting already has transport deadlines. Give healthy history/inventory hydration one
|
||||||
|
// shared budget instead of repeatedly discarding its progress on a short per-attempt timer.
|
||||||
|
let deadline = tokio::time::Instant::now() + Duration::from_secs(/*secs*/ 120);
|
||||||
|
for delay in [0, 1, 2, 4, 8] {
|
||||||
|
let attempt = async {
|
||||||
|
tokio::time::sleep(Duration::from_secs(delay)).await;
|
||||||
|
let client = crate::connect_remote_app_server(endpoint.clone()).await?;
|
||||||
|
let mut session = AppServerSession::new(client, mode)
|
||||||
|
.with_startup_config(&config)
|
||||||
|
.with_remote_cwd_override(remote_cwd.clone())
|
||||||
|
.with_thread_tool_transport(task_tools.clone());
|
||||||
|
let bootstrap = session.bootstrap(&config).await?;
|
||||||
|
let thread = if let Some(thread_id) = thread_id {
|
||||||
|
match session
|
||||||
|
.resume_thread(
|
||||||
|
config.clone(),
|
||||||
|
thread_id,
|
||||||
|
ResumeModelSettings::PreserveExistingThread,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(thread) => Some(thread),
|
||||||
|
Err(error)
|
||||||
|
if matches!(
|
||||||
|
error.downcast_ref::<TypedRequestError>(),
|
||||||
|
Some(TypedRequestError::Transport { .. })
|
||||||
|
) =>
|
||||||
|
{
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
Err(error)
|
||||||
|
if matches!(
|
||||||
|
error.downcast_ref::<TypedRequestError>(),
|
||||||
|
Some(TypedRequestError::Server { source, .. }) if source.code == -32600
|
||||||
|
) || presentation == ReconnectPresentation::Overview =>
|
||||||
|
{
|
||||||
|
None
|
||||||
|
}
|
||||||
|
Err(error) => return Err(error),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
Ok::<_, color_eyre::Report>(Reconnected {
|
||||||
|
session,
|
||||||
|
bootstrap,
|
||||||
|
thread,
|
||||||
|
})
|
||||||
|
};
|
||||||
|
let result = tokio::time::timeout_at(deadline, attempt).await;
|
||||||
|
match result {
|
||||||
|
Ok(Ok(connected)) => return Ok(connected),
|
||||||
|
Ok(Err(_)) => {}
|
||||||
|
Err(_) => break,
|
||||||
|
}
|
||||||
|
// Transport errors can contain endpoint credentials. Do not render or log them.
|
||||||
|
}
|
||||||
|
color_eyre::eyre::bail!("app-server session could not be restored")
|
||||||
|
}
|
||||||
|
|
||||||
impl App {
|
impl App {
|
||||||
|
pub(super) fn thread_unavailable(&self, id: ThreadId) -> bool {
|
||||||
|
!matches!(self.app_server_target, AppServerTarget::Embedded)
|
||||||
|
&& self
|
||||||
|
.thread_event_channels
|
||||||
|
.get(&id)
|
||||||
|
.is_some_and(|channel| channel.attachment() == ThreadEventAttachment::ReplayOnly)
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn recover_transport_error(&mut self, error: &color_eyre::Report) -> bool {
|
pub(super) fn recover_transport_error(&mut self, error: &color_eyre::Report) -> bool {
|
||||||
let disconnected = matches!(
|
let disconnected = matches!(
|
||||||
error.downcast_ref::<TypedRequestError>(),
|
error.downcast_ref::<TypedRequestError>(),
|
||||||
@@ -31,6 +135,7 @@ impl App {
|
|||||||
}
|
}
|
||||||
if !self.reconnect.offline {
|
if !self.reconnect.offline {
|
||||||
self.reconnect.offline = true;
|
self.reconnect.offline = true;
|
||||||
|
self.reconnect.failed = false;
|
||||||
self.cancel_pending_key_chord();
|
self.cancel_pending_key_chord();
|
||||||
self.overlay = None;
|
self.overlay = None;
|
||||||
self.commit_animation = None;
|
self.commit_animation = None;
|
||||||
@@ -63,4 +168,182 @@ impl App {
|
|||||||
}
|
}
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) async fn finish_reconnect(
|
||||||
|
&mut self,
|
||||||
|
tui: &mut tui::Tui,
|
||||||
|
app_server: &mut AppServerSession,
|
||||||
|
app_event_rx: &mut mpsc::UnboundedReceiver<AppEvent>,
|
||||||
|
connected: Reconnected,
|
||||||
|
) -> Result<()> {
|
||||||
|
let Reconnected {
|
||||||
|
mut session,
|
||||||
|
bootstrap,
|
||||||
|
thread,
|
||||||
|
} = connected;
|
||||||
|
let displayed = self.current_displayed_thread_id();
|
||||||
|
let mut input = self.chat_widget.capture_thread_input_state();
|
||||||
|
if let Some(input) = input.as_mut() {
|
||||||
|
input.recovered_queue = true;
|
||||||
|
}
|
||||||
|
self.store_active_thread_receiver().await;
|
||||||
|
// Old request handles stay attached to the dead connection. Rotating the event channel
|
||||||
|
// prevents their late completions from submitting follow-up operations on the new one.
|
||||||
|
let (tx, rx) = mpsc::unbounded_channel();
|
||||||
|
self.app_event_tx = AppEventSender::new(tx);
|
||||||
|
*app_event_rx = rx;
|
||||||
|
self.agent_navigation.picker_refresh = None;
|
||||||
|
self.last_subagent_backfill_attempt = None;
|
||||||
|
self.rate_limit_refresh_state.invalidate_recovery();
|
||||||
|
session.inherit_task_tool_capabilities(app_server);
|
||||||
|
*app_server = session;
|
||||||
|
self.chat_widget.remote_connection =
|
||||||
|
crate::status::remote_connection::remote_connection_status_value(
|
||||||
|
&self.app_server_target,
|
||||||
|
app_server.server_version(),
|
||||||
|
);
|
||||||
|
self.workspace_command_runner = Some(Arc::new(AppServerWorkspaceCommandRunner::new(
|
||||||
|
app_server.request_handle(),
|
||||||
|
)));
|
||||||
|
self.file_search =
|
||||||
|
FileSearchManager::new(self.config.cwd.to_path_buf(), self.app_event_tx.clone());
|
||||||
|
self.model_catalog = Arc::new(ModelCatalog::new(bootstrap.available_models));
|
||||||
|
self.pending_app_server_requests.clear();
|
||||||
|
self.pending_primary_events.clear();
|
||||||
|
self.pending_plugin_enabled_writes.clear();
|
||||||
|
self.pending_hook_enabled_writes.clear();
|
||||||
|
self.temporary_structured_requests.clear();
|
||||||
|
self.agents_overview.dispatched_requests.clear();
|
||||||
|
self.agents_overview.request_id = None;
|
||||||
|
self.agents_overview.refresh_pending = false;
|
||||||
|
for input in self.agents_overview.input_states.values_mut() {
|
||||||
|
input.recovered_queue = true;
|
||||||
|
}
|
||||||
|
self.pending_startup_thread_start = false;
|
||||||
|
// Move cached UI state into fresh channels. Old producers retain the old sender/store,
|
||||||
|
// so their late requests cannot leak into recovery. Background threads attach on selection.
|
||||||
|
for channel in self.thread_event_channels.values_mut() {
|
||||||
|
let mut replacement = ThreadEventChannel::new(THREAD_EVENT_CHANNEL_CAPACITY);
|
||||||
|
replacement.mark_replay_only();
|
||||||
|
let mut store = std::mem::replace(
|
||||||
|
&mut *channel.store.lock().await,
|
||||||
|
ThreadEventStore::new(THREAD_EVENT_CHANNEL_CAPACITY),
|
||||||
|
);
|
||||||
|
store.pending_interactive_replay = Default::default();
|
||||||
|
store.pending_interrupt_turn_id = None;
|
||||||
|
store.active = false;
|
||||||
|
store
|
||||||
|
.buffer
|
||||||
|
.retain(|event| matches!(event, ThreadBufferedEvent::Notification(_)));
|
||||||
|
if let Some(input) = store.input_state.as_mut() {
|
||||||
|
input.recovered_queue = true;
|
||||||
|
}
|
||||||
|
*replacement.store.lock().await = store;
|
||||||
|
*channel = replacement;
|
||||||
|
}
|
||||||
|
if let Some(id) = self.current_displayed_thread_id()
|
||||||
|
&& let Some(mut input) = input.clone()
|
||||||
|
{
|
||||||
|
input.recovered_queue = true;
|
||||||
|
self.agents_overview.input_states.insert(id, input);
|
||||||
|
}
|
||||||
|
if let Some(mut started) = thread {
|
||||||
|
let id = started.session.thread_id;
|
||||||
|
// A lost settings response can leave the server snapshot behind the user's choice.
|
||||||
|
// Side conversations can replace the app-wide overrides, so only restore choices
|
||||||
|
// that still match this conversation's own cached settings.
|
||||||
|
if (self.current_displayed_thread_id() == Some(id)
|
||||||
|
|| self.primary_thread_id == Some(id))
|
||||||
|
&& let Some(channel) = self.thread_event_channels.get(&id)
|
||||||
|
&& let Some(cached) = channel.store.lock().await.session.as_ref()
|
||||||
|
{
|
||||||
|
if let Some(policy) = self.runtime_approval_policy_override
|
||||||
|
&& policy == cached.approval_policy
|
||||||
|
{
|
||||||
|
started.session.approval_policy = policy;
|
||||||
|
}
|
||||||
|
if let Some(profile) = &self.runtime_permission_profile_override
|
||||||
|
&& profile.permission_profile == cached.permission_profile
|
||||||
|
&& profile.active_permission_profile == cached.active_permission_profile
|
||||||
|
&& self.config.approvals_reviewer == cached.approvals_reviewer
|
||||||
|
{
|
||||||
|
started.session.permission_profile = profile.permission_profile.clone();
|
||||||
|
started.session.active_permission_profile =
|
||||||
|
profile.active_permission_profile.clone();
|
||||||
|
started.session.approvals_reviewer = self.config.approvals_reviewer;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.agents_overview.input_states.remove(&id);
|
||||||
|
if started
|
||||||
|
.turns
|
||||||
|
.iter()
|
||||||
|
.any(|turn| turn.status == TurnStatus::InProgress)
|
||||||
|
{
|
||||||
|
self.agent_navigation.mark_running(id);
|
||||||
|
} else {
|
||||||
|
self.agent_navigation.mark_stopped(id);
|
||||||
|
}
|
||||||
|
if self.primary_thread_id == Some(id) {
|
||||||
|
self.primary_session_configured = Some(started.session.clone());
|
||||||
|
}
|
||||||
|
if started.blocks_direct_input {
|
||||||
|
self.agent_navigation.mark_parent_owned(id);
|
||||||
|
}
|
||||||
|
let channel = ThreadEventChannel::new(THREAD_EVENT_CHANNEL_CAPACITY);
|
||||||
|
{
|
||||||
|
let mut store = channel.store.lock().await;
|
||||||
|
store.set_session(started.session, started.turns);
|
||||||
|
store.input_state = input.clone();
|
||||||
|
}
|
||||||
|
self.thread_event_channels.insert(id, channel);
|
||||||
|
}
|
||||||
|
if let Some(id) = displayed
|
||||||
|
&& let Some((receiver, snapshot)) = self.activate_thread_for_replay(id).await
|
||||||
|
{
|
||||||
|
self.active_thread_id = Some(id);
|
||||||
|
self.active_thread_rx = Some(receiver);
|
||||||
|
self.recap.seed_from_turns(&snapshot.turns, Instant::now());
|
||||||
|
self.render_thread_snapshot(
|
||||||
|
tui, app_server, id, snapshot, /*resume_restored_queue*/ false,
|
||||||
|
)?;
|
||||||
|
self.config = self.chat_widget.config_ref().clone();
|
||||||
|
self.refresh_pending_thread_approvals().await;
|
||||||
|
if self.thread_unavailable(id) {
|
||||||
|
self.agent_navigation.mark_stopped(id);
|
||||||
|
self.chat_widget.pause_unavailable_thread();
|
||||||
|
self.chat_widget.add_info_message("This conversation is unavailable. Its cached transcript and draft remain here; input is paused. Open the agent picker or return to the parent to continue.".into(), /*hint*/ None);
|
||||||
|
} else {
|
||||||
|
self.recap
|
||||||
|
.schedule_check(id, self.app_event_tx.clone(), Instant::now());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
self.active_thread_id = None;
|
||||||
|
self.active_thread_rx = None;
|
||||||
|
self.primary_thread_id = None;
|
||||||
|
self.primary_session_configured = None;
|
||||||
|
let init = self.chatwidget_init_for_forked_or_resumed_thread(
|
||||||
|
tui,
|
||||||
|
self.config.clone(),
|
||||||
|
/*initial_user_message*/ None,
|
||||||
|
);
|
||||||
|
self.replace_chat_widget(ChatWidget::new_with_app_event(init));
|
||||||
|
self.chat_widget.restore_reconnected_input(input);
|
||||||
|
}
|
||||||
|
// Only accept fresh task-tool calls once this connection and its event queue are adopted.
|
||||||
|
if let ThreadToolTransport::Mcp(server) = app_server.thread_tool_transport() {
|
||||||
|
server.reconnect(app_server.request_handle(), self.app_event_tx.clone());
|
||||||
|
}
|
||||||
|
self.reconnect.offline = false;
|
||||||
|
self.chat_widget.update_account_state(
|
||||||
|
bootstrap.status_account_display,
|
||||||
|
bootstrap.plan_type,
|
||||||
|
bootstrap.has_chatgpt_account,
|
||||||
|
matches!(bootstrap.auth_mode, Some(TelemetryAuthMode::Chatgpt)),
|
||||||
|
);
|
||||||
|
self.feedback_audience = bootstrap.feedback_audience;
|
||||||
|
self.chat_widget.add_info_message(
|
||||||
|
"Reconnected. No input was resent. Review uncertain submissions before retrying; recovered queues remain paused.".into(), /*hint*/ None,
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -487,6 +487,7 @@ impl App {
|
|||||||
.agent_navigation
|
.agent_navigation
|
||||||
.get(&thread_id)
|
.get(&thread_id)
|
||||||
.is_some_and(|entry| entry.is_closed);
|
.is_some_and(|entry| entry.is_closed);
|
||||||
|
is_replay_only |= self.thread_unavailable(thread_id);
|
||||||
let mut attached_replay_only = false;
|
let mut attached_replay_only = false;
|
||||||
if self.should_attach_live_thread_for_selection(thread_id) {
|
if self.should_attach_live_thread_for_selection(thread_id) {
|
||||||
match self
|
match self
|
||||||
@@ -495,9 +496,7 @@ impl App {
|
|||||||
{
|
{
|
||||||
Ok(live_attached) => {
|
Ok(live_attached) => {
|
||||||
attached_replay_only = !live_attached;
|
attached_replay_only = !live_attached;
|
||||||
if attached_replay_only {
|
is_replay_only |= attached_replay_only;
|
||||||
is_replay_only = true;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
self.chat_widget.add_error_message(format!(
|
self.chat_widget.add_error_message(format!(
|
||||||
@@ -542,7 +541,6 @@ impl App {
|
|||||||
if snapshot.input_state.is_none() {
|
if snapshot.input_state.is_none() {
|
||||||
snapshot.input_state = self.agents_overview.input_states.remove(&thread_id);
|
snapshot.input_state = self.agents_overview.input_states.remove(&thread_id);
|
||||||
}
|
}
|
||||||
let blocks_direct_input = self.agent_navigation.is_parent_owned(thread_id);
|
|
||||||
|
|
||||||
self.active_thread_id = Some(thread_id);
|
self.active_thread_id = Some(thread_id);
|
||||||
self.active_thread_rx = Some(receiver);
|
self.active_thread_rx = Some(receiver);
|
||||||
@@ -558,23 +556,9 @@ impl App {
|
|||||||
self.recap
|
self.recap
|
||||||
.schedule_check(thread_id, self.app_event_tx.clone(), now);
|
.schedule_check(thread_id, self.app_event_tx.clone(), now);
|
||||||
|
|
||||||
let init = self.chatwidget_init_for_forked_or_resumed_thread(
|
self.render_thread_snapshot(tui, app_server, thread_id, snapshot, !is_replay_only)?;
|
||||||
tui,
|
|
||||||
self.config.clone(),
|
|
||||||
/*initial_user_message*/ None,
|
|
||||||
);
|
|
||||||
self.replace_chat_widget(ChatWidget::new_with_app_event(init));
|
|
||||||
self.chat_widget
|
|
||||||
.set_task_mentions_enabled(app_server.task_tools_available(thread_id));
|
|
||||||
self.chat_widget
|
|
||||||
.note_rendered_width(tui.terminal.last_known_screen_size.width);
|
|
||||||
if blocks_direct_input {
|
|
||||||
self.chat_widget.set_parent_owned_thread();
|
|
||||||
}
|
|
||||||
|
|
||||||
self.reset_for_thread_switch(tui)?;
|
|
||||||
self.replay_thread_snapshot(snapshot, !is_replay_only);
|
|
||||||
if is_replay_only {
|
if is_replay_only {
|
||||||
|
self.chat_widget.pause_unavailable_thread();
|
||||||
let message = if attached_replay_only {
|
let message = if attached_replay_only {
|
||||||
format!(
|
format!(
|
||||||
"Agent thread {thread_id} could not be resumed live. Replaying saved transcript."
|
"Agent thread {thread_id} could not be resumed live. Replaying saved transcript."
|
||||||
@@ -589,6 +573,32 @@ impl App {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn render_thread_snapshot(
|
||||||
|
&mut self,
|
||||||
|
tui: &mut tui::Tui,
|
||||||
|
app_server: &AppServerSession,
|
||||||
|
thread_id: ThreadId,
|
||||||
|
snapshot: ThreadEventSnapshot,
|
||||||
|
resume_restored_queue: bool,
|
||||||
|
) -> Result<()> {
|
||||||
|
let init = self.chatwidget_init_for_forked_or_resumed_thread(
|
||||||
|
tui,
|
||||||
|
self.config.clone(),
|
||||||
|
/*initial_user_message*/ None,
|
||||||
|
);
|
||||||
|
self.replace_chat_widget(ChatWidget::new_with_app_event(init));
|
||||||
|
self.chat_widget
|
||||||
|
.set_task_mentions_enabled(app_server.task_tools_available(thread_id));
|
||||||
|
self.chat_widget
|
||||||
|
.note_rendered_width(tui.terminal.last_known_screen_size.width);
|
||||||
|
if self.agent_navigation.is_parent_owned(thread_id) {
|
||||||
|
self.chat_widget.set_parent_owned_thread();
|
||||||
|
}
|
||||||
|
self.reset_for_thread_switch(tui)?;
|
||||||
|
self.replay_thread_snapshot(snapshot, resume_restored_queue);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn should_attach_live_thread_for_selection(&self, thread_id: ThreadId) -> bool {
|
pub(super) fn should_attach_live_thread_for_selection(&self, thread_id: ThreadId) -> bool {
|
||||||
!self.thread_event_channels.contains_key(&thread_id)
|
!self.thread_event_channels.contains_key(&thread_id)
|
||||||
&& self
|
&& self
|
||||||
|
|||||||
@@ -683,6 +683,7 @@ See the Codex keymap documentation for supported actions and examples."
|
|||||||
}
|
}
|
||||||
|
|
||||||
let mut listen_for_app_server_events = true;
|
let mut listen_for_app_server_events = true;
|
||||||
|
let mut reconnect = None;
|
||||||
let mut waiting_for_initial_session_configured = wait_for_initial_session_configured;
|
let mut waiting_for_initial_session_configured = wait_for_initial_session_configured;
|
||||||
let mut waiting_for_initial_session_header = true;
|
let mut waiting_for_initial_session_header = true;
|
||||||
|
|
||||||
@@ -711,6 +712,20 @@ See the Codex keymap documentation for supported actions and examples."
|
|||||||
Ok(exit_reason)
|
Ok(exit_reason)
|
||||||
} else {
|
} else {
|
||||||
loop {
|
loop {
|
||||||
|
if app.reconnect.offline
|
||||||
|
&& !app.reconnect.failed
|
||||||
|
&& reconnect.is_none()
|
||||||
|
&& app.reconnect.presentation == reconnect::ReconnectPresentation::Conversation
|
||||||
|
{
|
||||||
|
reconnect = Some(Box::pin(reconnect::reconnect(
|
||||||
|
app.app_server_target.clone(),
|
||||||
|
app.config.clone(),
|
||||||
|
app.current_displayed_thread_id(),
|
||||||
|
app_server.remote_cwd_override().map(Path::to_path_buf),
|
||||||
|
app_server.thread_tool_transport(),
|
||||||
|
app.reconnect.presentation,
|
||||||
|
)));
|
||||||
|
}
|
||||||
// Replay queues history and operations. A buffered closure must not switch
|
// Replay queues history and operations. A buffered closure must not switch
|
||||||
// widgets before those app events have been applied.
|
// widgets before those app events have been applied.
|
||||||
let has_pending_app_events = !app_event_rx.is_empty();
|
let has_pending_app_events = !app_event_rx.is_empty();
|
||||||
@@ -815,6 +830,25 @@ See the Codex keymap documentation for supported actions and examples."
|
|||||||
}
|
}
|
||||||
AppRunControl::Continue
|
AppRunControl::Continue
|
||||||
}
|
}
|
||||||
|
result = async { match reconnect.as_mut() { Some(future) => future.await, None => std::future::pending().await } }, if reconnect.is_some() && !has_pending_app_events => {
|
||||||
|
reconnect = None;
|
||||||
|
match result {
|
||||||
|
Ok(connected) => {
|
||||||
|
app.finish_reconnect(tui, &mut app_server, &mut app_event_rx, connected).await?;
|
||||||
|
listen_for_app_server_events = true;
|
||||||
|
waiting_for_initial_session_configured = false;
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
app.reconnect.failed = true;
|
||||||
|
app.chat_widget.reconnect_failed();
|
||||||
|
app.chat_widget.add_error_message(error.to_string());
|
||||||
|
if let Ok(mut state) = app.agents_overview.view_state.lock() {
|
||||||
|
state.connection_notice = Some("Reconnect failed — agent list is stale; relaunch to retry");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
AppRunControl::Continue
|
||||||
|
}
|
||||||
() = async {
|
() = async {
|
||||||
match app.chat_widget.terminal_title_next_refresh {
|
match app.chat_widget.terminal_title_next_refresh {
|
||||||
Some(deadline) => {
|
Some(deadline) => {
|
||||||
|
|||||||
@@ -8633,3 +8633,6 @@ async fn side_backtrack_rejection_reports_unavailable_message_snapshot() {
|
|||||||
async fn start_config_write_test_app_server(app: &App) -> Result<AppServerSession> {
|
async fn start_config_write_test_app_server(app: &App) -> Result<AppServerSession> {
|
||||||
Box::pin(crate::start_embedded_app_server_for_picker(&app.config)).await
|
Box::pin(crate::start_embedded_app_server_for_picker(&app.config)).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[path = "tests/active_reconnect_tests.rs"]
|
||||||
|
mod active_reconnect;
|
||||||
@@ -0,0 +1,462 @@
|
|||||||
|
//! Active recovery keeps local input and policy while reusing ordinary thread resume.
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::app::reconnect::ReconnectPresentation;
|
||||||
|
use crate::app::reconnect::reconnect;
|
||||||
|
use pretty_assertions::assert_eq;
|
||||||
|
use serde_json::json;
|
||||||
|
use tokio::net::TcpListener;
|
||||||
|
|
||||||
|
use super::disconnect::serve_reconnect_requests;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reconnect_restores_history_permissions_and_keeps_old_input_paused() -> Result<()> {
|
||||||
|
for (recovered_queue, edit_offline) in [(true, false), (false, false), (true, true)] {
|
||||||
|
let (mut app, mut events, mut ops) = make_test_app_with_channels().await;
|
||||||
|
let id = ThreadId::new();
|
||||||
|
let cwd = app.config.cwd.clone();
|
||||||
|
app.config.model = Some("gpt-test".into());
|
||||||
|
// Avoid platform-specific path widths in the mode-preservation snapshot.
|
||||||
|
app.config.tui_status_line = Some(vec!["model-with-reasoning".into()]);
|
||||||
|
app.config
|
||||||
|
.permissions
|
||||||
|
.set_permission_profile(PermissionProfile::read_only())?;
|
||||||
|
app.runtime_approval_policy_override = Some(AskForApproval::OnRequest);
|
||||||
|
app.runtime_permission_profile_override =
|
||||||
|
Some(RuntimePermissionProfileOverride::from_config(&app.config));
|
||||||
|
app.active_thread_id = Some(id);
|
||||||
|
app.primary_thread_id = Some(id);
|
||||||
|
let mut thread_session = test_thread_session(id, cwd.to_path_buf());
|
||||||
|
thread_session.approval_policy = AskForApproval::OnRequest;
|
||||||
|
thread_session.permission_profile = PermissionProfile::read_only();
|
||||||
|
app.ensure_thread_channel(id)
|
||||||
|
.store
|
||||||
|
.lock()
|
||||||
|
.await
|
||||||
|
.set_session(thread_session.clone(), Vec::new());
|
||||||
|
app.chat_widget.handle_thread_session(thread_session);
|
||||||
|
app.chat_widget.set_collaboration_mask(
|
||||||
|
crate::collaboration_modes::plan_mask(app.model_catalog.as_ref()).unwrap(),
|
||||||
|
);
|
||||||
|
let expected_mode = app.chat_widget.effective_collaboration_mode().with_updates(
|
||||||
|
Some("gpt-test".into()),
|
||||||
|
Some(None),
|
||||||
|
/*developer_instructions*/ None,
|
||||||
|
);
|
||||||
|
let expected_submitted_mode = expected_mode.clone();
|
||||||
|
if edit_offline {
|
||||||
|
app.chat_widget
|
||||||
|
.restore_user_message_to_composer("unacknowledged prompt".into());
|
||||||
|
app.chat_widget
|
||||||
|
.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE));
|
||||||
|
next_user_turn_op(&mut ops);
|
||||||
|
}
|
||||||
|
if recovered_queue {
|
||||||
|
app.chat_widget
|
||||||
|
.set_queue_autosend_suppressed(/*suppressed*/ true);
|
||||||
|
app.chat_widget
|
||||||
|
.restore_user_message_to_composer("old queued input".into());
|
||||||
|
app.chat_widget
|
||||||
|
.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE));
|
||||||
|
}
|
||||||
|
app.chat_widget
|
||||||
|
.restore_user_message_to_composer("kept draft".into());
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await?;
|
||||||
|
let endpoint = crate::resolve_remote_addr(&format!("ws://{}", listener.local_addr()?))?;
|
||||||
|
app.app_server_target = AppServerTarget::Remote { endpoint };
|
||||||
|
let thread = json!({
|
||||||
|
"id": id, "sessionId": id, "preview": "only once", "ephemeral": false,
|
||||||
|
"modelProvider": "test-provider", "createdAt": 1, "updatedAt": 2,
|
||||||
|
"status": {"type": "idle"}, "cwd": cwd, "cliVersion": "0.0.0", "source": "cli",
|
||||||
|
"turns": [test_turn("accepted", TurnStatus::Completed, vec![ThreadItem::UserMessage {
|
||||||
|
id: "input-1".into(), client_id: None, content: vec![UserInput::Text { text: "only once".into(), text_elements: Vec::new() }],
|
||||||
|
}])]
|
||||||
|
});
|
||||||
|
let server = tokio::spawn(async move {
|
||||||
|
let mut methods = Vec::new();
|
||||||
|
for attempt in 0..2 {
|
||||||
|
let (stream, _) = listener.accept().await?;
|
||||||
|
methods.extend(serve_reconnect_requests(tokio_tungstenite::accept_async(stream).await?, |request| std::future::ready(match request.method.as_str() {
|
||||||
|
"thread/resume" if attempt == 0 => Some(json!({"error": {"code": -32603, "message": "temporarily unavailable"}})),
|
||||||
|
"thread/resume" => {
|
||||||
|
let params = request.params.as_ref().unwrap();
|
||||||
|
assert_eq!(params["threadId"], id.to_string());
|
||||||
|
assert!(params["model"].is_null());
|
||||||
|
Some(json!({"result": {"thread": thread, "model": "gpt-test", "modelProvider": "test-provider", "cwd": cwd,
|
||||||
|
"approvalPolicy": "never", "approvalsReviewer": "user", "sandbox": {"type": "dangerFullAccess"}, "reasoningEffort": null}}))
|
||||||
|
}
|
||||||
|
"thread/read" => Some(json!({"result": {"thread": thread}})),
|
||||||
|
"thread/list" | "thread/loaded/list" => Some(json!({"result": {"data": [], "nextCursor": null}})),
|
||||||
|
"thread/goal/get" => Some(json!({"result": {"goal": null}})),
|
||||||
|
"turn/start" => {
|
||||||
|
assert!(!recovered_queue);
|
||||||
|
let params = request.params.as_ref().unwrap();
|
||||||
|
assert_eq!(params["input"][0]["text"], "fresh follow-up");
|
||||||
|
assert_eq!(params["approvalPolicy"], "on-request");
|
||||||
|
assert_eq!(params["sandboxPolicy"]["type"], "readOnly");
|
||||||
|
assert_eq!(params["collaborationMode"], json!(expected_submitted_mode));
|
||||||
|
Some(json!({"result": {"turn": {"id": "fresh", "items": [], "status": "inProgress"}}}))
|
||||||
|
}
|
||||||
|
method => panic!("unexpected reconnect request: {method}"),
|
||||||
|
})).await?);
|
||||||
|
}
|
||||||
|
Ok::<_, color_eyre::Report>(methods)
|
||||||
|
});
|
||||||
|
let mut session = crate::start_embedded_app_server_for_picker(&app.config).await?;
|
||||||
|
std::fs::write(
|
||||||
|
app.config
|
||||||
|
.codex_home
|
||||||
|
.join("tui-thread-reference-capabilities"),
|
||||||
|
"not a directory",
|
||||||
|
)?;
|
||||||
|
session.remember_task_tool_thread(id);
|
||||||
|
let transport = crate::dynamic_tools_mcp::ThreadToolTransport::Mcp(Arc::new(
|
||||||
|
crate::dynamic_tools_mcp::DynamicToolMcpServer::start(
|
||||||
|
session.request_handle(),
|
||||||
|
codex_app_server_protocol::ThreadStartParams::default(),
|
||||||
|
app.app_event_tx.clone(),
|
||||||
|
app.dynamic_tool_status_updates.clone(),
|
||||||
|
/*managed_requirement*/ None,
|
||||||
|
)
|
||||||
|
.await?,
|
||||||
|
));
|
||||||
|
let mut overrides = None;
|
||||||
|
transport.configure_mcp(&mut overrides);
|
||||||
|
let mcp = overrides.unwrap().remove("mcp_servers.codex_tui").unwrap();
|
||||||
|
let client = codex_http_client::HttpClientBuilder::new().build_direct()?;
|
||||||
|
let call_mcp = || {
|
||||||
|
client
|
||||||
|
.post(mcp["url"].as_str().unwrap())
|
||||||
|
.header(
|
||||||
|
"Authorization",
|
||||||
|
mcp["http_headers"]["Authorization"].as_str().unwrap(),
|
||||||
|
)
|
||||||
|
.header("Accept", "application/json, text/event-stream")
|
||||||
|
.header("MCP-Protocol-Version", "2026-07-28")
|
||||||
|
.header("MCP-Method", "tools/call")
|
||||||
|
.header("MCP-Name", "list_threads")
|
||||||
|
.json(
|
||||||
|
&json!({"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {
|
||||||
|
"name": "list_threads", "arguments": {}, "_meta": {"threadId": id,
|
||||||
|
"io.modelcontextprotocol/protocolVersion": "2026-07-28",
|
||||||
|
"io.modelcontextprotocol/clientCapabilities": {}}
|
||||||
|
}}),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
|
||||||
|
app.chat_widget.remote_connection =
|
||||||
|
crate::status::remote_connection::remote_connection_status_value(
|
||||||
|
&app.app_server_target,
|
||||||
|
Some("1.0.0"),
|
||||||
|
);
|
||||||
|
let mut tui = crate::tui::test_support::make_test_tui()?;
|
||||||
|
app.begin_reconnect();
|
||||||
|
if edit_offline {
|
||||||
|
app.handle_tui_event(
|
||||||
|
&mut tui,
|
||||||
|
&mut session,
|
||||||
|
TuiEvent::Key(KeyEvent::new(KeyCode::Up, KeyModifiers::ALT)),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
assert!(!app.chat_widget.has_queued_follow_up_messages());
|
||||||
|
assert!(
|
||||||
|
!app.chat_widget
|
||||||
|
.capture_thread_input_state()
|
||||||
|
.unwrap()
|
||||||
|
.recovered_queue
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let stale_picker_refresh = app.agent_navigation.begin_picker_refresh(id).unwrap();
|
||||||
|
app.last_subagent_backfill_attempt = Some(id);
|
||||||
|
app.rate_limit_refresh_state
|
||||||
|
.start(
|
||||||
|
RateLimitRefreshOrigin::Recovery,
|
||||||
|
&mut app.rate_limit_hard_stop_generation,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let before_disconnect = Instant::now() - Duration::from_secs(/*secs*/ 300);
|
||||||
|
app.recap.note_focus_lost(before_disconnect);
|
||||||
|
for _ in 0..3 {
|
||||||
|
app.recap
|
||||||
|
.note_turn_finished(&TurnStatus::Completed, before_disconnect);
|
||||||
|
}
|
||||||
|
app.recap
|
||||||
|
.schedule_check(id, app.app_event_tx.clone(), Instant::now());
|
||||||
|
let old_sender = app.app_event_tx.clone();
|
||||||
|
let connected = reconnect(
|
||||||
|
app.app_server_target.clone(),
|
||||||
|
app.config.clone(),
|
||||||
|
Some(id),
|
||||||
|
/*remote_cwd*/ None,
|
||||||
|
transport,
|
||||||
|
ReconnectPresentation::Conversation,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let paused = call_mcp().send().await?.text().await?;
|
||||||
|
assert!(
|
||||||
|
paused.contains("TUI is reconnecting; tool was not sent"),
|
||||||
|
"{paused}"
|
||||||
|
);
|
||||||
|
app.finish_reconnect(&mut tui, &mut session, &mut events, connected)
|
||||||
|
.await?;
|
||||||
|
assert!(!app.reconnect.offline);
|
||||||
|
assert_eq!(app.last_subagent_backfill_attempt, None);
|
||||||
|
assert!(
|
||||||
|
app.rate_limit_refresh_state
|
||||||
|
.start(
|
||||||
|
RateLimitRefreshOrigin::Recovery,
|
||||||
|
&mut app.rate_limit_hard_stop_generation
|
||||||
|
)
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
assert!(app.agent_navigation.begin_picker_refresh(id).is_some());
|
||||||
|
assert!(
|
||||||
|
!app.agent_navigation
|
||||||
|
.finish_picker_refresh(id, stale_picker_refresh)
|
||||||
|
);
|
||||||
|
let mut deferred = Vec::new();
|
||||||
|
tokio::time::timeout(Duration::from_secs(/*secs*/ 5), async {
|
||||||
|
loop {
|
||||||
|
let event = events.recv().await.expect("rebound recap event");
|
||||||
|
if matches!(event, AppEvent::CheckRecap { thread_id } if thread_id == id) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
deferred.push(event);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
for event in deferred {
|
||||||
|
app.app_event_tx.send(event);
|
||||||
|
}
|
||||||
|
let ready = call_mcp().send().await?.text().await?;
|
||||||
|
assert!(ready.contains("threads"), "{ready}");
|
||||||
|
assert_eq!(
|
||||||
|
app.chat_widget.remote_connection,
|
||||||
|
crate::status::remote_connection::remote_connection_status_value(
|
||||||
|
&app.app_server_target,
|
||||||
|
Some("2.0.0")
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert!(old_sender.app_event_tx.is_closed());
|
||||||
|
assert!(session.task_tools_available(id));
|
||||||
|
assert_eq!(app.current_displayed_thread_id(), Some(id));
|
||||||
|
assert_eq!(
|
||||||
|
app.chat_widget.effective_collaboration_mode(),
|
||||||
|
expected_mode
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
app.chat_widget.composer_text_with_pending(),
|
||||||
|
if edit_offline {
|
||||||
|
"old queued input"
|
||||||
|
} else {
|
||||||
|
"kept draft"
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let history = drain_history(&mut app, &mut tui, &mut session, &mut events).await?;
|
||||||
|
assert_eq!(history.matches("only once").count(), 1);
|
||||||
|
if recovered_queue {
|
||||||
|
assert!(app.chat_widget.has_queued_follow_up_messages());
|
||||||
|
assert!(!app.chat_widget.maybe_send_next_queued_input());
|
||||||
|
if edit_offline {
|
||||||
|
app.handle_tui_event(
|
||||||
|
&mut tui,
|
||||||
|
&mut session,
|
||||||
|
TuiEvent::Key(KeyEvent::new(KeyCode::Up, KeyModifiers::ALT)),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
assert_eq!(
|
||||||
|
app.chat_widget.composer_text_with_pending(),
|
||||||
|
"unacknowledged prompt"
|
||||||
|
);
|
||||||
|
assert!(ops.try_recv().is_err());
|
||||||
|
} else {
|
||||||
|
assert_snapshot!(
|
||||||
|
"restored_conversation",
|
||||||
|
render_bottom_popup(&app.chat_widget, /*width*/ 80)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
app.chat_widget
|
||||||
|
.handle_key_event(KeyEvent::new(KeyCode::Char('u'), KeyModifiers::CONTROL));
|
||||||
|
app.chat_widget
|
||||||
|
.restore_user_message_to_composer("fresh follow-up".into());
|
||||||
|
app.chat_widget
|
||||||
|
.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE));
|
||||||
|
while let Ok(event) = events.try_recv() {
|
||||||
|
app.handle_event(&mut tui, &mut session, event).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
session.shutdown().await?;
|
||||||
|
let methods = server.await??;
|
||||||
|
assert_eq!(
|
||||||
|
methods
|
||||||
|
.iter()
|
||||||
|
.filter(|method| *method == "turn/start")
|
||||||
|
.count(),
|
||||||
|
usize::from(!recovered_queue)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reconnect_exhaustion_and_unknown_initial_thread_stay_offline() -> Result<()> {
|
||||||
|
let (mut app, _, _) = make_test_app_with_channels().await;
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await?;
|
||||||
|
app.app_server_target = AppServerTarget::Remote {
|
||||||
|
endpoint: crate::resolve_remote_addr(&format!("ws://{}", listener.local_addr()?))?,
|
||||||
|
};
|
||||||
|
drop(listener);
|
||||||
|
tokio::time::pause();
|
||||||
|
let start = tokio::time::Instant::now();
|
||||||
|
for id in [Some(ThreadId::new()), None] {
|
||||||
|
assert!(
|
||||||
|
reconnect(
|
||||||
|
app.app_server_target.clone(),
|
||||||
|
app.config.clone(),
|
||||||
|
id,
|
||||||
|
/*remote_cwd*/ None,
|
||||||
|
crate::dynamic_tools_mcp::ThreadToolTransport::Dynamic,
|
||||||
|
ReconnectPresentation::Conversation
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!((15..=65).contains(&start.elapsed().as_secs()));
|
||||||
|
app.begin_reconnect();
|
||||||
|
app.chat_widget.reconnect_failed();
|
||||||
|
assert_snapshot!(
|
||||||
|
"reconnect_failed",
|
||||||
|
render_bottom_popup(&app.chat_widget, /*width*/ 80)
|
||||||
|
);
|
||||||
|
tokio::time::resume();
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reconnect_allows_slow_hydration_but_bounds_a_stalled_server() -> Result<()> {
|
||||||
|
for delay in [15, 150] {
|
||||||
|
let (mut app, mut events, _) = make_test_app_with_channels().await;
|
||||||
|
app.config.model = Some("gpt-test".into());
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await?;
|
||||||
|
let endpoint = crate::RemoteAppServerEndpoint::WebSocket {
|
||||||
|
websocket_url: format!("ws://{}", listener.local_addr()?),
|
||||||
|
auth_token: None,
|
||||||
|
};
|
||||||
|
let id = ThreadId::new();
|
||||||
|
let server = tokio::spawn(async move {
|
||||||
|
let (stream, _) = listener.accept().await?;
|
||||||
|
serve_reconnect_requests(
|
||||||
|
tokio_tungstenite::accept_async(stream).await?,
|
||||||
|
move |request| async move {
|
||||||
|
assert_eq!(request.method, "thread/resume");
|
||||||
|
tokio::time::pause();
|
||||||
|
tokio::time::sleep(Duration::from_secs(delay)).await;
|
||||||
|
tokio::time::resume();
|
||||||
|
Some(json!({"error": {"code": -32600, "message": "thread no longer exists"}}))
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
let start = tokio::time::Instant::now();
|
||||||
|
let result = reconnect(
|
||||||
|
AppServerTarget::Remote {
|
||||||
|
endpoint: endpoint.clone(),
|
||||||
|
},
|
||||||
|
app.config.clone(),
|
||||||
|
Some(id),
|
||||||
|
/*remote_cwd*/ None,
|
||||||
|
crate::dynamic_tools_mcp::ThreadToolTransport::Dynamic,
|
||||||
|
ReconnectPresentation::Conversation,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let elapsed = start.elapsed().as_secs();
|
||||||
|
if delay == 15 {
|
||||||
|
app.app_server_target = AppServerTarget::Remote { endpoint };
|
||||||
|
app.active_thread_id = Some(id);
|
||||||
|
app.primary_thread_id = Some(id);
|
||||||
|
let cached = test_thread_session(id, app.config.cwd.to_path_buf());
|
||||||
|
app.ensure_thread_channel(id)
|
||||||
|
.store
|
||||||
|
.lock()
|
||||||
|
.await
|
||||||
|
.set_session(cached, Vec::new());
|
||||||
|
app.chat_widget
|
||||||
|
.restore_user_message_to_composer("unavailable draft".into());
|
||||||
|
app.begin_reconnect();
|
||||||
|
let mut session = crate::start_embedded_app_server_for_picker(&app.config).await?;
|
||||||
|
let mut tui = crate::tui::test_support::make_test_tui()?;
|
||||||
|
app.finish_reconnect(&mut tui, &mut session, &mut events, result?)
|
||||||
|
.await?;
|
||||||
|
assert!(app.thread_unavailable(id));
|
||||||
|
app.handle_tui_event(
|
||||||
|
&mut tui,
|
||||||
|
&mut session,
|
||||||
|
TuiEvent::Key(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
assert_eq!(
|
||||||
|
app.chat_widget.composer_text_with_pending(),
|
||||||
|
"unavailable draft"
|
||||||
|
);
|
||||||
|
let history = drain_history(&mut app, &mut tui, &mut session, &mut events).await?;
|
||||||
|
let history = &history[history.find("• This conversation is unavailable").unwrap()..];
|
||||||
|
assert_snapshot!(
|
||||||
|
"unavailable_conversation",
|
||||||
|
format!(
|
||||||
|
"{history}\n{}",
|
||||||
|
render_bottom_popup(&app.chat_widget, /*width*/ 80)
|
||||||
|
)
|
||||||
|
.replace(
|
||||||
|
&test_path_buf("/tmp/project").display().to_string(),
|
||||||
|
"/tmp/project"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
session.shutdown().await?;
|
||||||
|
assert!((15..120).contains(&elapsed), "{elapsed}");
|
||||||
|
let methods = server.await??;
|
||||||
|
assert_eq!(
|
||||||
|
methods
|
||||||
|
.iter()
|
||||||
|
.filter(|method| *method == "initialize")
|
||||||
|
.count(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
tokio::time::resume();
|
||||||
|
assert!(result.is_err());
|
||||||
|
assert_eq!(elapsed, 120);
|
||||||
|
server.abort();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn drain_history(
|
||||||
|
app: &mut App,
|
||||||
|
tui: &mut tui::Tui,
|
||||||
|
session: &mut AppServerSession,
|
||||||
|
events: &mut mpsc::UnboundedReceiver<AppEvent>,
|
||||||
|
) -> Result<String> {
|
||||||
|
while let Ok(event) = events.try_recv() {
|
||||||
|
assert!(!matches!(
|
||||||
|
event,
|
||||||
|
AppEvent::CodexOp(AppCommand::UserTurn { .. })
|
||||||
|
));
|
||||||
|
if matches!(
|
||||||
|
event,
|
||||||
|
AppEvent::InsertHistoryCell(_)
|
||||||
|
| AppEvent::BeginThreadSwitchHistoryReplayBuffer
|
||||||
|
| AppEvent::EndInitialHistoryReplayBuffer
|
||||||
|
) {
|
||||||
|
app.handle_event(tui, session, event).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(lines_to_single_string(
|
||||||
|
&app.transcript_cells
|
||||||
|
.iter()
|
||||||
|
.flat_map(|cell| cell.display_lines(/*width*/ 80))
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
))
|
||||||
|
}
|
||||||
@@ -264,7 +264,7 @@ where
|
|||||||
methods.push(request.method.clone());
|
methods.push(request.method.clone());
|
||||||
let request_id = request.id.clone();
|
let request_id = request.id.clone();
|
||||||
let response = match request.method.as_str() {
|
let response = match request.method.as_str() {
|
||||||
"initialize" => Some(json!({"result": {"userAgent": "reconnect-test"}})),
|
"initialize" => Some(json!({"result": {"userAgent": "reconnect-test/2.0.0"}})),
|
||||||
"account/read" => {
|
"account/read" => {
|
||||||
Some(json!({"result": {"account": null, "requiresOpenaiAuth": false}}))
|
Some(json!({"result": {"account": null, "requiresOpenaiAuth": false}}))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -601,7 +601,7 @@ async fn archive_current_thread_reports_success_only_after_archiving() -> Result
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn local_daemon_registers_approval_gated_mcp_tools_for_both_start_paths() -> Result<()> {
|
async fn local_daemon_registers_approval_gated_mcp_tools_for_both_start_paths() -> Result<()> {
|
||||||
let (mut app, mut events, _ops) = make_test_app_with_channels().await;
|
let (mut app, events, _ops) = make_test_app_with_channels().await;
|
||||||
let codex_home = tempdir()?;
|
let codex_home = tempdir()?;
|
||||||
app.config.codex_home = codex_home.path().to_path_buf().abs();
|
app.config.codex_home = codex_home.path().to_path_buf().abs();
|
||||||
app.config.sqlite = SqliteConfig::new_for_testing(codex_home.path().abs());
|
app.config.sqlite = SqliteConfig::new_for_testing(codex_home.path().abs());
|
||||||
@@ -612,7 +612,7 @@ async fn local_daemon_registers_approval_gated_mcp_tools_for_both_start_paths()
|
|||||||
codex_home.path().join("config.toml"),
|
codex_home.path().join("config.toml"),
|
||||||
"web_search = \"disabled\"\n",
|
"web_search = \"disabled\"\n",
|
||||||
)?;
|
)?;
|
||||||
let (mut app_server, requests, proxy) = start_recording_app_server(
|
let (mut app_server, mut requests, mut proxy) = start_recording_app_server(
|
||||||
&app.config,
|
&app.config,
|
||||||
/*blocked_thread_list*/ None,
|
/*blocked_thread_list*/ None,
|
||||||
/*failed_thread_name*/ None,
|
/*failed_thread_name*/ None,
|
||||||
@@ -776,6 +776,41 @@ async fn local_daemon_registers_approval_gated_mcp_tools_for_both_start_paths()
|
|||||||
}
|
}
|
||||||
}))
|
}))
|
||||||
};
|
};
|
||||||
|
let transport = app_server.thread_tool_transport();
|
||||||
|
let crate::dynamic_tools_mcp::ThreadToolTransport::Mcp(tool_server) = &transport else {
|
||||||
|
panic!("expected the daemon task-tool bridge");
|
||||||
|
};
|
||||||
|
tool_server.suspend();
|
||||||
|
let paused = call_tool(0, "list_threads", serde_json::json!({}))
|
||||||
|
.send()
|
||||||
|
.await?
|
||||||
|
.text()
|
||||||
|
.await?;
|
||||||
|
assert!(
|
||||||
|
paused.contains("TUI is reconnecting; tool was not sent"),
|
||||||
|
"{paused}"
|
||||||
|
);
|
||||||
|
let (replacement, replacement_requests, replacement_proxy) = start_recording_app_server(
|
||||||
|
&app.config,
|
||||||
|
/*blocked_thread_list*/ None,
|
||||||
|
/*failed_thread_name*/ None,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let (new_tx, new_rx) = mpsc::unbounded_channel();
|
||||||
|
let new_sender = AppEventSender::new(new_tx);
|
||||||
|
drop(events);
|
||||||
|
let mut events = new_rx;
|
||||||
|
assert!(app.app_event_tx.app_event_tx.is_closed());
|
||||||
|
tool_server.reconnect(replacement.request_handle(), new_sender);
|
||||||
|
let previous = std::mem::replace(
|
||||||
|
&mut app_server,
|
||||||
|
replacement.with_thread_tool_transport(transport),
|
||||||
|
);
|
||||||
|
previous.shutdown().await?;
|
||||||
|
proxy.await??;
|
||||||
|
requests = replacement_requests;
|
||||||
|
proxy = replacement_proxy;
|
||||||
|
// The same MCP URL and credentials now use the new connection and event receiver.
|
||||||
let response = call_tool(1, "list_threads", serde_json::json!({}))
|
let response = call_tool(1, "list_threads", serde_json::json!({}))
|
||||||
.send()
|
.send()
|
||||||
.await?;
|
.await?;
|
||||||
@@ -824,9 +859,13 @@ async fn local_daemon_registers_approval_gated_mcp_tools_for_both_start_paths()
|
|||||||
child["config"]["mcp_servers.codex_tui"],
|
child["config"]["mcp_servers.codex_tui"],
|
||||||
starts[0]["config"]["mcp_servers.codex_tui"]
|
starts[0]["config"]["mcp_servers.codex_tui"]
|
||||||
);
|
);
|
||||||
let forked = call_tool(3, "fork_thread", serde_json::json!({"threadId": thread_id}))
|
let forked = call_tool(
|
||||||
.send()
|
3,
|
||||||
.await?;
|
"fork_thread",
|
||||||
|
serde_json::json!({"threadId": delegation_source}),
|
||||||
|
)
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
assert!(forked.status().is_success());
|
assert!(forked.status().is_success());
|
||||||
let forked = recorded_params(&requests, "thread/fork")
|
let forked = recorded_params(&requests, "thread/fork")
|
||||||
.pop()
|
.pop()
|
||||||
|
|||||||
+10
@@ -0,0 +1,10 @@
|
|||||||
|
---
|
||||||
|
source: tui/src/app/tests/active_reconnect_tests.rs
|
||||||
|
expression: "render_bottom_popup(&app.chat_widget, 80)"
|
||||||
|
---
|
||||||
|
• Reconnect failed — check the endpoint, then relaunch (0s)
|
||||||
|
|
||||||
|
|
||||||
|
› Ask Codex to do anything
|
||||||
|
|
||||||
|
ctrl+c quit
|
||||||
+11
@@ -0,0 +1,11 @@
|
|||||||
|
---
|
||||||
|
source: tui/src/app/tests/active_reconnect_tests.rs
|
||||||
|
expression: "render_bottom_popup(&app.chat_widget, 80)"
|
||||||
|
---
|
||||||
|
• Queued follow-up inputs
|
||||||
|
↳ old queued input
|
||||||
|
⌥ + ↑ edit last queued message
|
||||||
|
|
||||||
|
› kept draft
|
||||||
|
|
||||||
|
gpt-test default Plan mode (shift+tab to cycle)
|
||||||
+9
@@ -0,0 +1,9 @@
|
|||||||
|
---
|
||||||
|
source: tui/src/app/tests/active_reconnect_tests.rs
|
||||||
|
expression: "format!(\"{history}\\n{}\", render_bottom_popup(&app.chat_widget, 80))"
|
||||||
|
---
|
||||||
|
• This conversation is unavailable. Its cached transcript and draft remain here; input is paused. Open the agent picker or return to the parent to continue.
|
||||||
|
• Reconnected. No input was resent. Review uncertain submissions before retrying; recovered queues remain paused.
|
||||||
|
› unavailable draft
|
||||||
|
|
||||||
|
gpt-test default · /tmp/project
|
||||||
+2
-2
@@ -2,8 +2,8 @@
|
|||||||
source: tui/src/app/tests/disconnect_tests.rs
|
source: tui/src/app/tests/disconnect_tests.rs
|
||||||
expression: "format!(\"{}\\n{}\",\nlines_to_single_string(&app.transcript_cells.last().unwrap().display_lines(80)),\nrender_bottom_popup(&app.chat_widget, 80))"
|
expression: "format!(\"{}\\n{}\",\nlines_to_single_string(&app.transcript_cells.last().unwrap().display_lines(80)),\nrender_bottom_popup(&app.chat_widget, 80))"
|
||||||
---
|
---
|
||||||
■ Connection lost.
|
■ Connection lost. Attempting to reconnect…
|
||||||
• Connection lost — copy your draft, then relaunch (0s)
|
• Reconnecting to app-server… (0s)
|
||||||
|
|
||||||
• Queued follow-up inputs
|
• Queued follow-up inputs
|
||||||
↳ queued before disconnect
|
↳ queued before disconnect
|
||||||
|
|||||||
@@ -443,6 +443,12 @@ impl App {
|
|||||||
thread_id: ThreadId,
|
thread_id: ThreadId,
|
||||||
op: AppCommand,
|
op: AppCommand,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
|
if self.thread_unavailable(thread_id) {
|
||||||
|
self.chat_widget.add_error_message(
|
||||||
|
"This conversation is unavailable; no operation was sent.".into(),
|
||||||
|
);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
if self.chat_widget.rejects_misalignment_policy_op(&op) {
|
if self.chat_widget.rejects_misalignment_policy_op(&op) {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
@@ -1546,7 +1552,7 @@ impl App {
|
|||||||
|
|
||||||
pub(super) fn replay_thread_snapshot(
|
pub(super) fn replay_thread_snapshot(
|
||||||
&mut self,
|
&mut self,
|
||||||
snapshot: ThreadEventSnapshot,
|
mut snapshot: ThreadEventSnapshot,
|
||||||
resume_restored_queue: bool,
|
resume_restored_queue: bool,
|
||||||
) {
|
) {
|
||||||
let request_changes = snapshot
|
let request_changes = snapshot
|
||||||
@@ -1591,6 +1597,12 @@ impl App {
|
|||||||
self.chat_widget.handle_thread_session(session);
|
self.chat_widget.handle_thread_session(session);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
let recovered_input = snapshot
|
||||||
|
.input_state
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|input| input.recovered_queue)
|
||||||
|
.then(|| snapshot.input_state.take())
|
||||||
|
.flatten();
|
||||||
self.chat_widget.restore_thread_input_state(
|
self.chat_widget.restore_thread_input_state(
|
||||||
snapshot.input_state,
|
snapshot.input_state,
|
||||||
ThreadInputStateRestoreMode {
|
ThreadInputStateRestoreMode {
|
||||||
@@ -1616,6 +1628,9 @@ impl App {
|
|||||||
self.app_event_tx
|
self.app_event_tx
|
||||||
.send(AppEvent::EndInitialHistoryReplayBuffer);
|
.send(AppEvent::EndInitialHistoryReplayBuffer);
|
||||||
}
|
}
|
||||||
|
if recovered_input.is_some() {
|
||||||
|
self.chat_widget.restore_reconnected_input(recovered_input);
|
||||||
|
}
|
||||||
self.chat_widget
|
self.chat_widget
|
||||||
.set_queue_autosend_suppressed(/*suppressed*/ false);
|
.set_queue_autosend_suppressed(/*suppressed*/ false);
|
||||||
self.chat_widget
|
self.chat_widget
|
||||||
|
|||||||
@@ -464,6 +464,13 @@ impl AppServerSession {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn with_thread_tool_transport(mut self, transport: ThreadToolTransport) -> Self {
|
||||||
|
if let ThreadToolTransport::Mcp(server) = transport {
|
||||||
|
self.dynamic_tool_mcp = Some(server);
|
||||||
|
}
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn with_remote_cwd_override(mut self, remote_cwd_override: Option<PathBuf>) -> Self {
|
pub(crate) fn with_remote_cwd_override(mut self, remote_cwd_override: Option<PathBuf>) -> Self {
|
||||||
self.remote_cwd_override = remote_cwd_override;
|
self.remote_cwd_override = remote_cwd_override;
|
||||||
self
|
self
|
||||||
@@ -481,6 +488,11 @@ impl AppServerSession {
|
|||||||
matches!(&self.client, AppServerClient::InProcess(_))
|
matches!(&self.client, AppServerClient::InProcess(_))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Carry capabilities that may exist only in memory when the optional cache is unwritable.
|
||||||
|
pub(crate) fn inherit_task_tool_capabilities(&mut self, previous: &Self) {
|
||||||
|
self.task_tool_threads.extend(&previous.task_tool_threads);
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn task_tools_available(&self, thread_id: ThreadId) -> bool {
|
pub(crate) fn task_tools_available(&self, thread_id: ThreadId) -> bool {
|
||||||
self.task_tool_threads.contains(&thread_id)
|
self.task_tool_threads.contains(&thread_id)
|
||||||
|| self
|
|| self
|
||||||
|
|||||||
@@ -13,9 +13,47 @@ impl ChatWidget {
|
|||||||
self.bottom_pane.ensure_status_indicator();
|
self.bottom_pane.ensure_status_indicator();
|
||||||
self.bottom_pane
|
self.bottom_pane
|
||||||
.set_interrupt_hint_visible(/*visible*/ false);
|
.set_interrupt_hint_visible(/*visible*/ false);
|
||||||
self.set_status_header("Connection lost — copy your draft, then relaunch".to_string());
|
self.set_status_header("Reconnecting to app-server…".to_string());
|
||||||
self.set_footer_hint_override(Some(vec![("ctrl+c".into(), "quit".into())]));
|
self.set_footer_hint_override(Some(vec![("ctrl+c".into(), "quit".into())]));
|
||||||
self.add_error_message("Connection lost.".into());
|
self.add_error_message("Connection lost. Attempting to reconnect…".into());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Restore local input only after replay, which can otherwise move interrupted queues into the draft.
|
||||||
|
pub(crate) fn restore_reconnected_input(&mut self, input: Option<ThreadInputState>) {
|
||||||
|
let running = self.turn_lifecycle.agent_turn_running;
|
||||||
|
if let Some(mut input) = input {
|
||||||
|
// Its acceptance is unknown. Keep a local copy for manual recovery without
|
||||||
|
// comparing against partial history or automatically submitting it again.
|
||||||
|
if input.user_turn_pending_start
|
||||||
|
&& let Some(prompt) = input.safety_buffering_prompt.take()
|
||||||
|
{
|
||||||
|
input.queued_user_messages.push_front(prompt.into());
|
||||||
|
input
|
||||||
|
.queued_user_message_history_records
|
||||||
|
.push_front(UserMessageHistoryRecord::UserMessageText);
|
||||||
|
input.recovered_queue = true;
|
||||||
|
}
|
||||||
|
input.current_collaboration_mode = self.current_collaboration_mode.clone();
|
||||||
|
// Resume supplies model/effort, but not the user's selected collaboration mode.
|
||||||
|
if let Some(mask) = input.active_collaboration_mask.as_mut() {
|
||||||
|
mask.model = Some(self.current_model().to_string());
|
||||||
|
mask.reasoning_effort = Some(self.effective_reasoning_effort());
|
||||||
|
}
|
||||||
|
self.restore_thread_input_state(
|
||||||
|
Some(input),
|
||||||
|
ThreadInputStateRestoreMode {
|
||||||
|
preserve_in_flight_turn: false,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
self.turn_lifecycle.restore_running(running, Instant::now());
|
||||||
|
self.update_task_running_state();
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn pause_unavailable_thread(&mut self) {
|
||||||
|
self.turn_lifecycle
|
||||||
|
.restore_running(/*running*/ false, Instant::now());
|
||||||
|
self.update_task_running_state();
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn handle_disconnected_view_key(&mut self, key: KeyEvent) {
|
pub(crate) fn handle_disconnected_view_key(&mut self, key: KeyEvent) {
|
||||||
@@ -41,4 +79,11 @@ impl ChatWidget {
|
|||||||
self.bottom_pane.handle_disconnected_key(key);
|
self.bottom_pane.handle_disconnected_key(key);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn reconnect_failed(&mut self) {
|
||||||
|
self.set_status_header("Reconnect failed — check the endpoint, then relaunch".into());
|
||||||
|
self.add_error_message(
|
||||||
|
"Automatic reconnect could not restore this session. Your draft is still editable. Copy it before quitting with Ctrl+C, then reconnect with the same command.".into(),
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -2413,3 +2413,40 @@ async fn interrupt_prepends_queued_messages_before_existing_composer_text() {
|
|||||||
|
|
||||||
let _ = drain_insert_history(&mut rx);
|
let _ = drain_insert_history(&mut rx);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reconnect_holds_only_recovered_input_until_manually_edited() {
|
||||||
|
for (recovered, pending_start) in [
|
||||||
|
(None, false),
|
||||||
|
(Some("review this old input"), false),
|
||||||
|
(Some("unacknowledged prompt"), true),
|
||||||
|
] {
|
||||||
|
let (mut chat, _rx, _ops) = make_chatwidget_manual(/*model_override*/ None).await;
|
||||||
|
if let Some(text) = recovered {
|
||||||
|
if pending_start {
|
||||||
|
chat.input_queue.user_turn_pending_start = true;
|
||||||
|
chat.safety_buffering_prompt = Some(UserMessage::from(text));
|
||||||
|
} else {
|
||||||
|
chat.input_queue
|
||||||
|
.queued_user_messages
|
||||||
|
.push_back(UserMessage::from(text).into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
chat.pause_for_disconnect();
|
||||||
|
let input = chat.capture_thread_input_state();
|
||||||
|
let (mut chat, _rx, mut ops) = make_chatwidget_manual(/*model_override*/ None).await;
|
||||||
|
chat.thread_id = Some(ThreadId::new());
|
||||||
|
chat.restore_reconnected_input(input);
|
||||||
|
chat.set_queue_autosend_suppressed(/*suppressed*/ false);
|
||||||
|
if let Some(text) = recovered {
|
||||||
|
assert!(!chat.maybe_send_next_queued_input());
|
||||||
|
assert_eq!(chat.pop_latest_queued_composer_state().unwrap().text, text);
|
||||||
|
assert_no_submit_op(&mut ops);
|
||||||
|
}
|
||||||
|
chat.input_queue
|
||||||
|
.queued_user_messages
|
||||||
|
.push_back(UserMessage::from("new follow-up").into());
|
||||||
|
assert!(chat.maybe_send_next_queued_input());
|
||||||
|
assert_matches!(next_submit_op(&mut ops), Op::UserTurn { .. });
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -616,6 +616,7 @@ async fn execute_inner(
|
|||||||
thread_start_params.sandbox = None;
|
thread_start_params.sandbox = None;
|
||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
|
thread_start_params.permissions = None;
|
||||||
thread_start_params.sandbox = Some(match &source.sandbox {
|
thread_start_params.sandbox = Some(match &source.sandbox {
|
||||||
SandboxPolicy::DangerFullAccess => SandboxMode::DangerFullAccess,
|
SandboxPolicy::DangerFullAccess => SandboxMode::DangerFullAccess,
|
||||||
SandboxPolicy::ReadOnly { .. } => SandboxMode::ReadOnly,
|
SandboxPolicy::ReadOnly { .. } => SandboxMode::ReadOnly,
|
||||||
|
|||||||
@@ -43,6 +43,8 @@ use serde_json::json;
|
|||||||
use std::borrow::Cow;
|
use std::borrow::Cow;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::sync::PoisonError;
|
||||||
|
use std::sync::RwLock;
|
||||||
use tokio::net::TcpListener;
|
use tokio::net::TcpListener;
|
||||||
use tokio::sync::broadcast;
|
use tokio::sync::broadcast;
|
||||||
use tokio::task::JoinHandle;
|
use tokio::task::JoinHandle;
|
||||||
@@ -79,12 +81,29 @@ impl ThreadToolTransport {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ToolConnection = Arc<RwLock<Option<(AppServerRequestHandle, AppEventSender)>>>;
|
||||||
|
|
||||||
pub(crate) struct DynamicToolMcpServer {
|
pub(crate) struct DynamicToolMcpServer {
|
||||||
|
connection: ToolConnection,
|
||||||
config: Value,
|
config: Value,
|
||||||
task: JoinHandle<()>,
|
task: JoinHandle<()>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DynamicToolMcpServer {
|
impl DynamicToolMcpServer {
|
||||||
|
pub(crate) fn suspend(&self) {
|
||||||
|
*self
|
||||||
|
.connection
|
||||||
|
.write()
|
||||||
|
.unwrap_or_else(PoisonError::into_inner) = None;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn reconnect(&self, handle: AppServerRequestHandle, events: AppEventSender) {
|
||||||
|
*self
|
||||||
|
.connection
|
||||||
|
.write()
|
||||||
|
.unwrap_or_else(PoisonError::into_inner) = Some((handle, events));
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn start(
|
pub(crate) async fn start(
|
||||||
request_handle: AppServerRequestHandle,
|
request_handle: AppServerRequestHandle,
|
||||||
mut thread_start_params: ThreadStartParams,
|
mut thread_start_params: ThreadStartParams,
|
||||||
@@ -122,10 +141,10 @@ impl DynamicToolMcpServer {
|
|||||||
if let Some(overrides) = thread_start_params.config.as_mut() {
|
if let Some(overrides) = thread_start_params.config.as_mut() {
|
||||||
overrides.remove("web_search");
|
overrides.remove("web_search");
|
||||||
}
|
}
|
||||||
|
let connection = Arc::new(RwLock::new(Some((request_handle, app_event_tx))));
|
||||||
let handler = DynamicToolMcpHandler {
|
let handler = DynamicToolMcpHandler {
|
||||||
request_handle,
|
connection: Arc::clone(&connection),
|
||||||
thread_start_params,
|
thread_start_params,
|
||||||
app_event_tx,
|
|
||||||
status_updates,
|
status_updates,
|
||||||
server_config: server_config.clone(),
|
server_config: server_config.clone(),
|
||||||
};
|
};
|
||||||
@@ -147,6 +166,7 @@ impl DynamicToolMcpServer {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
|
connection,
|
||||||
config: server_config,
|
config: server_config,
|
||||||
task,
|
task,
|
||||||
})
|
})
|
||||||
@@ -177,9 +197,8 @@ async fn require_authorization(
|
|||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
struct DynamicToolMcpHandler {
|
struct DynamicToolMcpHandler {
|
||||||
request_handle: AppServerRequestHandle,
|
connection: ToolConnection,
|
||||||
thread_start_params: ThreadStartParams,
|
thread_start_params: ThreadStartParams,
|
||||||
app_event_tx: AppEventSender,
|
|
||||||
status_updates: broadcast::Sender<ThreadStatusChangedNotification>,
|
status_updates: broadcast::Sender<ThreadStatusChangedNotification>,
|
||||||
server_config: Value,
|
server_config: Value,
|
||||||
}
|
}
|
||||||
@@ -265,12 +284,21 @@ impl ServerHandler for DynamicToolMcpHandler {
|
|||||||
format!("mcp_servers.{}", dynamic_tools::NAMESPACE),
|
format!("mcp_servers.{}", dynamic_tools::NAMESPACE),
|
||||||
self.server_config.clone(),
|
self.server_config.clone(),
|
||||||
);
|
);
|
||||||
|
// Snapshot once: an in-flight call must never switch connections or replay a mutation.
|
||||||
|
let (request_handle, app_event_tx) = self
|
||||||
|
.connection
|
||||||
|
.read()
|
||||||
|
.unwrap_or_else(PoisonError::into_inner)
|
||||||
|
.clone()
|
||||||
|
.ok_or_else(|| {
|
||||||
|
McpError::internal_error("TUI is reconnecting; tool was not sent", None)
|
||||||
|
})?;
|
||||||
let response = dynamic_tools::execute(
|
let response = dynamic_tools::execute(
|
||||||
self.request_handle.clone(),
|
request_handle,
|
||||||
params,
|
params,
|
||||||
thread_start_params,
|
thread_start_params,
|
||||||
self.status_updates.subscribe(),
|
self.status_updates.subscribe(),
|
||||||
Some(&self.app_event_tx),
|
Some(&app_event_tx),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let content = response
|
let content = response
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
//! Drives disconnect handling through the real binary and terminal event loop.
|
//! Drives automatic reconnect through the real binary and terminal event loop.
|
||||||
|
|
||||||
use super::focus_palette::PtyCodex;
|
use super::focus_palette::PtyCodex;
|
||||||
use super::focus_palette::write_test_config;
|
use super::focus_palette::write_test_config;
|
||||||
@@ -15,7 +15,7 @@ use tokio::net::UnixListener;
|
|||||||
use tokio_tungstenite::tungstenite::Message;
|
use tokio_tungstenite::tungstenite::Message;
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
async fn disconnect_preserves_editable_draft_in_terminal() -> Result<()> {
|
async fn automatic_reconnect_restores_draft_and_routes_new_notifications() -> Result<()> {
|
||||||
let repo_root = codex_utils_cargo_bin::repo_root()?;
|
let repo_root = codex_utils_cargo_bin::repo_root()?;
|
||||||
// macOS's default temporary directory leaves too little room for the control socket path.
|
// macOS's default temporary directory leaves too little room for the control socket path.
|
||||||
let codex_home = tempfile::tempdir_in("/tmp")?;
|
let codex_home = tempfile::tempdir_in("/tmp")?;
|
||||||
@@ -24,9 +24,11 @@ async fn disconnect_preserves_editable_draft_in_terminal() -> Result<()> {
|
|||||||
std::fs::create_dir_all(socket.parent().unwrap())?;
|
std::fs::create_dir_all(socket.parent().unwrap())?;
|
||||||
let listener = UnixListener::bind(socket.as_path())?;
|
let listener = UnixListener::bind(socket.as_path())?;
|
||||||
let (disconnect_tx, mut disconnect_rx) = tokio::sync::oneshot::channel();
|
let (disconnect_tx, mut disconnect_rx) = tokio::sync::oneshot::channel();
|
||||||
|
let (restore_tx, restore_rx) = tokio::sync::oneshot::channel();
|
||||||
let server_cwd = repo_root.clone();
|
let server_cwd = repo_root.clone();
|
||||||
let server = tokio::spawn(async move {
|
let server = tokio::spawn(async move {
|
||||||
let mut methods = Vec::new();
|
let mut methods = Vec::new();
|
||||||
|
let mut restore_rx = Some(restore_rx);
|
||||||
let id = "00000000-0000-0000-0000-000000000001";
|
let id = "00000000-0000-0000-0000-000000000001";
|
||||||
let thread = json!({
|
let thread = json!({
|
||||||
"id": id, "sessionId": id, "preview": "", "ephemeral": false,
|
"id": id, "sessionId": id, "preview": "", "ephemeral": false,
|
||||||
@@ -34,7 +36,7 @@ async fn disconnect_preserves_editable_draft_in_terminal() -> Result<()> {
|
|||||||
"status": {"type": "active", "activeFlags": []}, "cwd": server_cwd,
|
"status": {"type": "active", "activeFlags": []}, "cwd": server_cwd,
|
||||||
"cliVersion": "0.0.0", "source": "cli", "turns": [{"id": "running", "items": [], "status": "inProgress", "error": null}]
|
"cliVersion": "0.0.0", "source": "cli", "turns": [{"id": "running", "items": [], "status": "inProgress", "error": null}]
|
||||||
});
|
});
|
||||||
for connection in 0..1 {
|
for connection in 0..2 {
|
||||||
let mut socket = loop {
|
let mut socket = loop {
|
||||||
let (stream, _) = listener.accept().await?;
|
let (stream, _) = listener.accept().await?;
|
||||||
// Startup probes the default daemon socket before opening its WebSocket.
|
// Startup probes the default daemon socket before opening its WebSocket.
|
||||||
@@ -57,6 +59,9 @@ async fn disconnect_preserves_editable_draft_in_terminal() -> Result<()> {
|
|||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
methods.push(request.method.clone());
|
methods.push(request.method.clone());
|
||||||
|
if connection == 1 && request.method == "initialize" {
|
||||||
|
restore_rx.take().unwrap().await?;
|
||||||
|
}
|
||||||
let result = match request.method.as_str() {
|
let result = match request.method.as_str() {
|
||||||
"initialize" => json!({"userAgent": "reconnect-pty"}),
|
"initialize" => json!({"userAgent": "reconnect-pty"}),
|
||||||
"account/read" => {
|
"account/read" => {
|
||||||
@@ -92,6 +97,22 @@ async fn disconnect_preserves_editable_draft_in_terminal() -> Result<()> {
|
|||||||
.into(),
|
.into(),
|
||||||
))
|
))
|
||||||
.await?;
|
.await?;
|
||||||
|
if connection == 1 && request.method == "thread/resume" {
|
||||||
|
// Keep the recovered turn running: its output must appear without waiting
|
||||||
|
// for turn/completed or rebuilding the transcript.
|
||||||
|
socket
|
||||||
|
.send(Message::Text(
|
||||||
|
json!({
|
||||||
|
"method": "item/agentMessage/delta", "params": {
|
||||||
|
"threadId": id, "turnId": "running", "itemId": "live-item",
|
||||||
|
"delta": "fresh-notification-after-reconnect\n"
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.to_string()
|
||||||
|
.into(),
|
||||||
|
))
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok::<_, anyhow::Error>(methods)
|
Ok::<_, anyhow::Error>(methods)
|
||||||
@@ -99,11 +120,13 @@ async fn disconnect_preserves_editable_draft_in_terminal() -> Result<()> {
|
|||||||
let mut terminal = PtyCodex::start(&repo_root, codex_home)?;
|
let mut terminal = PtyCodex::start(&repo_root, codex_home)?;
|
||||||
terminal.wait_for_startup()?;
|
terminal.wait_for_startup()?;
|
||||||
let mut disconnect_tx = Some(disconnect_tx);
|
let mut disconnect_tx = Some(disconnect_tx);
|
||||||
|
let mut restore_tx = Some(restore_tx);
|
||||||
for expected in [
|
for expected in [
|
||||||
"gpt-5.6-terra",
|
"gpt-5.6-terra",
|
||||||
"preserved-draft",
|
"preserved-draft",
|
||||||
"Connection lost",
|
"Reconnecting",
|
||||||
"preserved-draft!",
|
"preserved-draft!",
|
||||||
|
"fresh-notification-after-reconnect",
|
||||||
] {
|
] {
|
||||||
let deadline = Instant::now() + Duration::from_secs(/*secs*/ 30);
|
let deadline = Instant::now() + Duration::from_secs(/*secs*/ 30);
|
||||||
while !terminal.screen_contains(expected) && Instant::now() < deadline {
|
while !terminal.screen_contains(expected) && Instant::now() < deadline {
|
||||||
@@ -119,7 +142,10 @@ async fn disconnect_preserves_editable_draft_in_terminal() -> Result<()> {
|
|||||||
"preserved-draft" => {
|
"preserved-draft" => {
|
||||||
disconnect_tx.take().unwrap().send(()).unwrap();
|
disconnect_tx.take().unwrap().send(()).unwrap();
|
||||||
}
|
}
|
||||||
"Connection lost" => terminal.write_input(b"!")?,
|
"Reconnecting" => terminal.write_input(b"!")?,
|
||||||
|
"preserved-draft!" => {
|
||||||
|
restore_tx.take().unwrap().send(()).unwrap();
|
||||||
|
}
|
||||||
_ => {}
|
_ => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -134,7 +160,7 @@ async fn disconnect_preserves_editable_draft_in_terminal() -> Result<()> {
|
|||||||
.iter()
|
.iter()
|
||||||
.filter(|method| *method == "thread/resume")
|
.filter(|method| *method == "thread/resume")
|
||||||
.count(),
|
.count(),
|
||||||
0
|
1
|
||||||
);
|
);
|
||||||
assert!(!methods.iter().any(|method| method == "turn/start"));
|
assert!(!methods.iter().any(|method| method == "turn/start"));
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|||||||
Reference in new issue
Block a user