mirror of
https://github.com/openai/codex.git
synced 2026-11-14 06:48:26 +00:00
Cancel remote control enrollment on stdio shutdown (#42668)
## Why A pending remote control enrollment could prevent the app server from exiting after stdio EOF, leaving resources such as thread writers held by the process. ## What changed - Give remote control its own child shutdown token and cancel it before draining RPCs when the stdio connection closes. - Interrupt enrollment requests that require network access after shutdown has begun, while still allowing in-memory or persisted enrollments to be enabled and durably saved. ## Testing - Cover stdio shutdown during a blocked enrollment and verify that another app server can acquire the released thread writer. - Cover durable enablement after shutdown with in-memory, persisted, and missing enrollments. GitOrigin-RevId: ef9ab49672f273a4cf8257188454a154c03088bd
This commit is contained in:
1 parent
ff2f01b0c2
commit
ea2046f36d
5 files changed
+347
-9
No files matched your search
@@ -106,6 +106,7 @@ pub(super) struct QueuedServerEnvelope {
|
||||
#[derive(Clone)]
|
||||
pub struct RemoteControlHandle {
|
||||
policy: RemoteControlPolicy,
|
||||
shutdown_token: CancellationToken,
|
||||
desired_state_tx: Arc<watch::Sender<RemoteControlDesiredState>>,
|
||||
desired_state_rpc_lock: Arc<Semaphore>,
|
||||
desired_state_persistence_lock: Arc<Semaphore>,
|
||||
@@ -592,14 +593,23 @@ impl RemoteControlHandle {
|
||||
RemoteControlEnrollmentSelection::ReplaceExisting => {}
|
||||
}
|
||||
|
||||
let enrollment = enroll_pairing_server(
|
||||
&self.auth_manager,
|
||||
auth,
|
||||
&remote_control_target,
|
||||
installation_id,
|
||||
server_name,
|
||||
)
|
||||
.await?;
|
||||
// Reused enrollments must still reach durable persistence during shutdown.
|
||||
let enrollment = tokio::select! {
|
||||
biased;
|
||||
_ = self.shutdown_token.cancelled() => {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::Interrupted,
|
||||
"remote control is shutting down",
|
||||
));
|
||||
}
|
||||
result = enroll_pairing_server(
|
||||
&self.auth_manager,
|
||||
auth,
|
||||
&remote_control_target,
|
||||
installation_id,
|
||||
server_name,
|
||||
) => result?,
|
||||
};
|
||||
Ok((enrollment, true))
|
||||
}
|
||||
|
||||
@@ -1005,6 +1015,7 @@ pub async fn start_remote_control(
|
||||
let installation_id_for_log = installation_id.clone();
|
||||
let server_name_for_log = server_name.clone();
|
||||
let shutdown_token_for_log = shutdown_token.clone();
|
||||
let handle_shutdown_token = shutdown_token.clone();
|
||||
let join_handle = tokio::spawn(async move {
|
||||
info!(
|
||||
remote_control_url = %remote_control_url_for_log,
|
||||
@@ -1070,6 +1081,7 @@ pub async fn start_remote_control(
|
||||
join_handle,
|
||||
RemoteControlHandle {
|
||||
policy,
|
||||
shutdown_token: handle_shutdown_token,
|
||||
desired_state_tx,
|
||||
desired_state_rpc_lock,
|
||||
desired_state_persistence_lock,
|
||||
|
||||
@@ -411,6 +411,7 @@ pub(super) fn remote_control_handle_with_current_enrollment(
|
||||
)));
|
||||
RemoteControlHandle {
|
||||
policy: RemoteControlPolicy::Allowed,
|
||||
shutdown_token: CancellationToken::new(),
|
||||
desired_state_tx: Arc::new(desired_state_tx),
|
||||
desired_state_rpc_lock: Arc::new(Semaphore::new(1)),
|
||||
desired_state_persistence_lock: Arc::new(Semaphore::new(1)),
|
||||
@@ -424,6 +425,216 @@ pub(super) fn remote_control_handle_with_current_enrollment(
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn durable_enable_reuses_in_memory_enrollment_after_shutdown() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("listener should bind");
|
||||
let remote_control_url = remote_control_url_for_listener(&listener);
|
||||
let codex_home = TempDir::new().expect("temp dir should create");
|
||||
let state_db = remote_control_state_runtime(&codex_home).await;
|
||||
let mut remote_handle = remote_control_handle_with_current_enrollment(
|
||||
&remote_control_url,
|
||||
remote_control_auth_manager(),
|
||||
);
|
||||
remote_handle.state_db = Some(state_db.clone());
|
||||
remote_handle
|
||||
.desired_state_tx
|
||||
.send_replace(RemoteControlDesiredState::Disabled);
|
||||
let enrollment = remote_handle
|
||||
.current_enrollment
|
||||
.snapshot()
|
||||
.expect("in-memory enrollment should exist");
|
||||
let expected_record = RemoteControlEnrollmentRecord {
|
||||
websocket_url: enrollment.remote_control_target.websocket_url.clone(),
|
||||
account_id: enrollment.account_id.clone(),
|
||||
app_server_client_name: None,
|
||||
server_id: enrollment.server_id.clone(),
|
||||
environment_id: enrollment.environment_id.clone(),
|
||||
server_name: enrollment.server_name.clone(),
|
||||
remote_control_enabled: Some(true),
|
||||
};
|
||||
assert_eq!(
|
||||
state_db
|
||||
.get_remote_control_enrollment(
|
||||
&expected_record.websocket_url,
|
||||
&expected_record.account_id,
|
||||
/*app_server_client_name*/ None,
|
||||
)
|
||||
.await
|
||||
.expect("enrollment should load"),
|
||||
None
|
||||
);
|
||||
remote_handle.shutdown_token.cancel();
|
||||
|
||||
let status = timeout(
|
||||
Duration::from_secs(5),
|
||||
remote_handle.enable(/*app_server_client_name*/ None),
|
||||
)
|
||||
.await
|
||||
.expect("cached enable should complete without network I/O")
|
||||
.expect("shutdown should not cancel durable enable using in-memory enrollment");
|
||||
|
||||
assert_eq!(
|
||||
state_db
|
||||
.get_remote_control_enrollment(
|
||||
&expected_record.websocket_url,
|
||||
&expected_record.account_id,
|
||||
/*app_server_client_name*/ None,
|
||||
)
|
||||
.await
|
||||
.expect("enabled enrollment should load"),
|
||||
Some(expected_record)
|
||||
);
|
||||
assert_eq!(
|
||||
*remote_handle.desired_state_tx.borrow(),
|
||||
RemoteControlDesiredState::Enabled {
|
||||
persistence_preference: Some(true),
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
status.environment_id.as_deref(),
|
||||
Some(enrollment.environment_id.as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
remote_handle.current_enrollment.snapshot(),
|
||||
Some(enrollment)
|
||||
);
|
||||
timeout(Duration::from_millis(100), listener.accept())
|
||||
.await
|
||||
.expect_err("in-memory enrollment should prevent backend contact");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn durable_enable_reuses_persisted_enrollment_after_shutdown() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("listener should bind");
|
||||
let remote_control_url = remote_control_url_for_listener(&listener);
|
||||
let remote_control_target = normalize_remote_control_url(&remote_control_url)
|
||||
.expect("remote control target should normalize");
|
||||
let codex_home = TempDir::new().expect("temp dir should create");
|
||||
let state_db = remote_control_state_runtime(&codex_home).await;
|
||||
let persisted_enrollment = RemoteControlEnrollmentRecord {
|
||||
websocket_url: remote_control_target.websocket_url,
|
||||
account_id: "account_id".to_string(),
|
||||
app_server_client_name: None,
|
||||
server_id: "persisted-server-id".to_string(),
|
||||
environment_id: "persisted-environment-id".to_string(),
|
||||
server_name: format!("{}-persisted", test_server_name()),
|
||||
remote_control_enabled: Some(false),
|
||||
};
|
||||
state_db
|
||||
.upsert_remote_control_enrollment(&persisted_enrollment)
|
||||
.await
|
||||
.expect("disabled enrollment should persist");
|
||||
let mut remote_handle = remote_control_handle_with_current_enrollment(
|
||||
&remote_control_url,
|
||||
remote_control_auth_manager(),
|
||||
);
|
||||
remote_handle.state_db = Some(state_db.clone());
|
||||
*remote_handle.current_enrollment.lock().await = None;
|
||||
remote_handle
|
||||
.desired_state_tx
|
||||
.send_replace(RemoteControlDesiredState::Disabled);
|
||||
remote_handle.shutdown_token.cancel();
|
||||
|
||||
let status = timeout(
|
||||
Duration::from_secs(5),
|
||||
remote_handle.enable(/*app_server_client_name*/ None),
|
||||
)
|
||||
.await
|
||||
.expect("cached enable should complete without network I/O")
|
||||
.expect("shutdown should not cancel durable enable using persisted enrollment");
|
||||
|
||||
assert_eq!(
|
||||
status.environment_id.as_deref(),
|
||||
Some(persisted_enrollment.environment_id.as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
remote_handle
|
||||
.current_enrollment
|
||||
.snapshot()
|
||||
.map(|enrollment| enrollment.server_id),
|
||||
Some(persisted_enrollment.server_id.clone())
|
||||
);
|
||||
assert_eq!(
|
||||
state_db
|
||||
.get_remote_control_enrollment(
|
||||
&persisted_enrollment.websocket_url,
|
||||
&persisted_enrollment.account_id,
|
||||
/*app_server_client_name*/ None,
|
||||
)
|
||||
.await
|
||||
.expect("enabled enrollment should load"),
|
||||
Some(RemoteControlEnrollmentRecord {
|
||||
remote_control_enabled: Some(true),
|
||||
..persisted_enrollment
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
*remote_handle.desired_state_tx.borrow(),
|
||||
RemoteControlDesiredState::Enabled {
|
||||
persistence_preference: Some(true),
|
||||
}
|
||||
);
|
||||
timeout(Duration::from_millis(100), listener.accept())
|
||||
.await
|
||||
.expect_err("persisted enrollment should prevent backend contact");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn durable_enable_without_cached_enrollment_is_cancelled_after_shutdown() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("listener should bind");
|
||||
let remote_control_url = remote_control_url_for_listener(&listener);
|
||||
let remote_control_target = normalize_remote_control_url(&remote_control_url)
|
||||
.expect("remote control target should normalize");
|
||||
let codex_home = TempDir::new().expect("temp dir should create");
|
||||
let state_db = remote_control_state_runtime(&codex_home).await;
|
||||
let mut remote_handle = remote_control_handle_with_current_enrollment(
|
||||
&remote_control_url,
|
||||
remote_control_auth_manager(),
|
||||
);
|
||||
remote_handle.state_db = Some(state_db.clone());
|
||||
*remote_handle.current_enrollment.lock().await = None;
|
||||
remote_handle
|
||||
.desired_state_tx
|
||||
.send_replace(RemoteControlDesiredState::Disabled);
|
||||
remote_handle.shutdown_token.cancel();
|
||||
|
||||
let error = timeout(
|
||||
Duration::from_secs(5),
|
||||
remote_handle.enable(/*app_server_client_name*/ None),
|
||||
)
|
||||
.await
|
||||
.expect("shutdown should cancel network enrollment promptly")
|
||||
.expect_err("enable without cached enrollment should be cancelled");
|
||||
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::Interrupted);
|
||||
assert_eq!(error.to_string(), "remote control is shutting down");
|
||||
assert_eq!(remote_handle.current_enrollment.snapshot(), None);
|
||||
assert_eq!(
|
||||
*remote_handle.desired_state_tx.borrow(),
|
||||
RemoteControlDesiredState::Disabled
|
||||
);
|
||||
assert_eq!(
|
||||
state_db
|
||||
.get_remote_control_enrollment(
|
||||
&remote_control_target.websocket_url,
|
||||
"account_id",
|
||||
/*app_server_client_name*/ None,
|
||||
)
|
||||
.await
|
||||
.expect("enrollment should load"),
|
||||
None
|
||||
);
|
||||
timeout(Duration::from_millis(100), listener.accept())
|
||||
.await
|
||||
.expect_err("cancelled enrollment should prevent backend contact");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ephemeral_enable_preserves_durable_preference() {
|
||||
let codex_home = TempDir::new().expect("temp dir should create");
|
||||
|
||||
@@ -23,6 +23,7 @@ fn client_management_handle(
|
||||
});
|
||||
RemoteControlHandle {
|
||||
policy: RemoteControlPolicy::Allowed,
|
||||
shutdown_token: CancellationToken::new(),
|
||||
desired_state_tx: Arc::new(desired_state_tx),
|
||||
desired_state_rpc_lock: Arc::new(Semaphore::new(1)),
|
||||
desired_state_persistence_lock: Arc::new(Semaphore::new(1)),
|
||||
|
||||
Reference in new issue
Block a user