//! Best-effort thread persistence during managed app-server shutdown. use super::connection_handling_websocket::DEFAULT_READ_TIMEOUT; use super::connection_handling_websocket::create_config_toml; use anyhow::Context; use anyhow::Result; use app_test_support::DISABLE_PLUGIN_STARTUP_TASKS_ARG; use app_test_support::create_final_assistant_message_sse_response; use codex_app_server_protocol::InitializeCapabilities; use codex_app_server_protocol::JSONRPCMessage; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ThreadStartResponse; use codex_uds::UnixStream; use core_test_support::streaming_sse::StreamingSseChunk; use core_test_support::streaming_sse::start_streaming_sse_server; use futures::SinkExt; use futures::StreamExt; use pretty_assertions::assert_eq; use serde_json::json; use std::path::Path; #[cfg(unix)] use std::process::Command as StdCommand; use std::process::Stdio; use tempfile::TempDir; use tokio::process::Child; use tokio::process::Command; use tokio::sync::oneshot; use tokio::time::Duration; use tokio::time::sleep; use tokio::time::timeout; use tokio_tungstenite::WebSocketStream; use tokio_tungstenite::client_async; use tokio_tungstenite::tungstenite::Message; #[tokio::test] async fn managed_shutdown_persists_loaded_threads_after_active_turn() -> Result<()> { let home = TempDir::new()?; let (release_response, response_gate) = oneshot::channel(); let (mock, _completions) = start_streaming_sse_server(vec![vec![stream_chunk(Some(response_gate), "Done")?]]).await; create_config_toml(home.path(), mock.uri(), "never")?; let socket_path = home.path().join("control/server.sock"); let mut server = spawn_server(home.path(), &socket_path)?; let mut client = connect_default_daemon_client(&socket_path).await?; let active = start_thread(&mut client, /*id*/ 2, json!({})).await?; let idle = start_thread(&mut client, /*id*/ 3, json!({})).await?; start_turn(&mut client, /*id*/ 4, &active.thread.id).await?; wait_for_requests(&mock, /*count*/ 1).await?; request_shutdown(&server, &socket_path).await?; assert_still_running(&mut server, "active turn must finish first").await; release_response.send(()).expect("response is waiting"); wait_success(&mut server).await?; let mut successor = spawn_server(home.path(), &socket_path)?; let mut reconnected = connect_default_daemon_client(&socket_path).await?; for (id, thread) in [(2, &active.thread), (3, &idle.thread)] { let resumed = request( &mut reconnected, id, "thread/resume", json!({"threadId":thread.id}), ) .await?; assert_eq!(resumed["thread"]["id"], thread.id); if thread.id == active.thread.id { assert!( resumed["thread"]["turns"][0]["items"] .as_array() .context("recovered turn items")? .iter() .any(|item| item["type"] == "agentMessage" && item["text"] == "Done") ); } } request_shutdown(&successor, &socket_path).await?; wait_success(&mut successor).await?; Ok(()) } #[tokio::test] async fn managed_force_shutdown_exits_with_active_work() -> Result<()> { let home = TempDir::new()?; let (_release_response, response_gate) = oneshot::channel(); let (mock, _completions) = start_streaming_sse_server(vec![vec![stream_chunk(Some(response_gate), "Done")?]]).await; create_config_toml(home.path(), mock.uri(), "never")?; let socket_path = home.path().join("control/server.sock"); let mut server = spawn_server(home.path(), &socket_path)?; let mut client = connect_default_daemon_client(&socket_path).await?; let thread = start_thread(&mut client, /*id*/ 2, json!({})).await?; start_turn(&mut client, /*id*/ 3, &thread.thread.id).await?; wait_for_requests(&mock, /*count*/ 1).await?; request_shutdown(&server, &socket_path).await?; assert_still_running(&mut server, "graceful shutdown must wait").await; request_shutdown(&server, &socket_path).await?; wait_success(&mut server).await?; Ok(()) } #[cfg(unix)] #[tokio::test] async fn managed_force_shutdown_exits_with_blocked_rollout_writer() -> Result<()> { let home = TempDir::new()?; let mock = wiremock::MockServer::start().await; create_config_toml(home.path(), &mock.uri(), "never")?; let socket_path = home.path().join("control/server.sock"); let mut server = spawn_server(home.path(), &socket_path)?; let mut client = connect_default_daemon_client(&socket_path).await?; let thread = start_thread(&mut client, /*id*/ 2, json!({})).await?; let rollout = thread .thread .path .context("persistent thread rollout path")?; assert!(!rollout.exists(), "idle rollout should still be deferred"); let compressed = rollout.with_extension("jsonl.zst"); std::fs::create_dir_all(compressed.parent().context("rollout directory")?)?; assert!( StdCommand::new("mkfifo") .arg(&compressed) .status()? .success() ); request_shutdown(&server, &socket_path).await?; // Opening the write end succeeds only once the real rollout writer opens // the compressed input. Keep it empty and open to block that writer's read. let _blocked_input = timeout(Duration::from_secs(10), async { loop { if let Ok(sender) = tokio::net::unix::pipe::OpenOptions::new().open_sender(&compressed) { return sender; } sleep(Duration::from_millis(25)).await; } }) .await .context("rollout writer did not open the blocked input")?; assert_still_running(&mut server, "graceful shutdown must wait for the writer").await; request_shutdown(&server, &socket_path).await?; wait_success(&mut server).await?; Ok(()) } async fn wait_success(server: &mut Child) -> Result<()> { anyhow::ensure!( timeout(Duration::from_secs(10), server.wait()) .await?? .success(), "server did not exit successfully" ); Ok(()) } async fn assert_still_running(server: &mut Child, message: &str) { assert!( timeout(Duration::from_millis(300), server.wait()) .await .is_err(), "{message}" ); } fn stream_chunk(gate: Option>, body: &str) -> Result { Ok(StreamingSseChunk { gate, body: create_final_assistant_message_sse_response(body)?, }) } async fn wait_for_requests( mock: &core_test_support::streaming_sse::StreamingSseServer, count: usize, ) -> Result<()> { timeout(Duration::from_secs(5), mock.wait_for_request_count(count)).await?; Ok(()) } async fn start_turn( websocket: &mut WebSocketStream, id: i64, thread_id: impl serde::Serialize, ) -> Result<()> { request( websocket, id, "turn/start", json!({"threadId":thread_id,"input":[{"type":"text","text":"Hello"}]}), ) .await?; Ok(()) } async fn start_thread( websocket: &mut WebSocketStream, id: i64, params: serde_json::Value, ) -> Result { Ok(serde_json::from_value( request(websocket, id, "thread/start", params).await?, )?) } async fn connect_default_daemon_client(socket_path: &Path) -> Result> { connect_daemon_client(socket_path, InitializeCapabilities::default()).await } async fn connect_daemon_client( socket_path: &Path, capabilities: InitializeCapabilities, ) -> Result> { connect_initialized(socket_path, capabilities, "daemon_recovery_test", "0.1.0").await } fn spawn_server(home: &Path, socket_path: &Path) -> Result { let binary = codex_utils_cargo_bin::cargo_bin("codex-app-server")?; Ok(Command::new(binary) .args(["--listen", &format!("unix://{}", socket_path.display())]) .arg(DISABLE_PLUGIN_STARTUP_TASKS_ARG) .env("CODEX_HOME", home) .arg("--managed-daemon") .env( codex_app_server_transport::DAEMON_SHUTDOWN_SOCKET_ENV, socket_path, ) .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::null()) .kill_on_drop(true) .spawn()?) } #[cfg(unix)] async fn request_shutdown(server: &Child, _socket_path: &Path) -> Result<()> { let pid = server.id().context("server pid")?; let status = StdCommand::new("kill") .arg("-TERM") .arg(pid.to_string()) .status()?; anyhow::ensure!(status.success(), "failed to signal server {pid}: {status}"); Ok(()) } #[cfg(windows)] async fn request_shutdown(server: &Child, socket_path: &Path) -> Result<()> { let pid = server.id().context("server pid")?.to_string(); timeout(DEFAULT_READ_TIMEOUT, async { let stream = UnixStream::connect(socket_path).await?; let (mut websocket, _) = client_async("ws://localhost/daemon/shutdown", stream).await?; websocket.send(Message::Text(pid.clone().into())).await?; let reply = websocket .next() .await .context("missing shutdown acknowledgment")??; assert_eq!(reply, Message::Text(pid.into())); websocket.close(None).await?; Ok(()) }) .await? } async fn connect_initialized( socket_path: &Path, capabilities: InitializeCapabilities, client_name: &str, client_version: &str, ) -> Result> { let mut websocket = timeout(DEFAULT_READ_TIMEOUT, async { loop { if let Ok(stream) = UnixStream::connect(socket_path).await && let Ok((websocket, _)) = client_async("ws://localhost/", stream).await { return websocket; } sleep(Duration::from_millis(25)).await; } }) .await .context("server did not open its control socket")?; request( &mut websocket, /*id*/ 1, "initialize", json!({"clientInfo":{"name":client_name,"version":client_version},"capabilities":capabilities}), ) .await?; websocket .send(Message::Text( json!({"method":"initialized"}).to_string().into(), )) .await?; Ok(websocket) } async fn request( websocket: &mut WebSocketStream, id: i64, method: &str, params: serde_json::Value, ) -> Result { websocket .send(Message::Text( json!({"id":id,"method":method,"params":params}) .to_string() .into(), )) .await?; timeout(DEFAULT_READ_TIMEOUT, async { loop { let frame = websocket.next().await.context("socket closed")??; let Message::Text(text) = frame else { continue }; match serde_json::from_str::(&text)? { JSONRPCMessage::Response(response) if response.id == RequestId::Integer(id) => { return Ok(response.result); } JSONRPCMessage::Error(error) if error.id == RequestId::Integer(id) => { anyhow::bail!("{}: {}", method, error.error.message); } _ => {} } } }) .await .context("timed out waiting for app-server response")? }