mirror of
https://github.com/openai/codex.git
synced 2026-11-16 06:54:36 +00:00
Handle pending network reviews after process completion (#42746)
## Why A remote process can finish while a network policy review is still pending. Normal process cleanup should withdraw that review without turning the completed command into a review failure or losing its output. ## What changed - Record whether a network policy request was withdrawn because the process finished, was cancelled, lost its executor connection, or timed out. - Treat normal process completion as cleanup while retaining fail-closed behavior for other cancellation causes. - Preserve explicit network denials before policy persistence so cleanup cannot replace the reported call outcome. ## Testing Add an integration test that completes a remote process during a pending network review and verifies that the command reports its successful exit and output without approving the withdrawn request. GitOrigin-RevId: 7f42d75631ee29eba43bf04cc953eea490f553fc
This commit is contained in:
1 parent
4e48cd02da
commit
8e85265c39
9 files changed
+430
-11
No files matched your search
@@ -786,6 +786,7 @@ async fn network_approval_uses_published_task_authority_within_same_turn(
|
|||||||
exec_policy_hint: None,
|
exec_policy_hint: None,
|
||||||
execution_id: None,
|
execution_id: None,
|
||||||
disconnect: None,
|
disconnect: None,
|
||||||
|
cancellation: None,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
tokio::pin!(decision);
|
tokio::pin!(decision);
|
||||||
|
|||||||
@@ -17,6 +17,8 @@ use codex_network_proxy::NetworkPolicyDecider;
|
|||||||
use codex_network_proxy::NetworkPolicyRequest;
|
use codex_network_proxy::NetworkPolicyRequest;
|
||||||
use codex_network_proxy::NetworkProtocol;
|
use codex_network_proxy::NetworkProtocol;
|
||||||
use codex_network_proxy::NetworkProxy;
|
use codex_network_proxy::NetworkProxy;
|
||||||
|
use codex_network_proxy::NetworkRequestCancellation;
|
||||||
|
use codex_network_proxy::NetworkRequestCancellationReason;
|
||||||
use codex_network_proxy::NetworkRequestDisconnect;
|
use codex_network_proxy::NetworkRequestDisconnect;
|
||||||
use codex_protocol::approvals::NetworkApprovalContext;
|
use codex_protocol::approvals::NetworkApprovalContext;
|
||||||
use codex_protocol::approvals::NetworkApprovalProtocol;
|
use codex_protocol::approvals::NetworkApprovalProtocol;
|
||||||
@@ -280,6 +282,7 @@ struct PendingHostApprovalOwner<'a> {
|
|||||||
pending: Arc<PendingHostApproval>,
|
pending: Arc<PendingHostApproval>,
|
||||||
execution_cancellation: Option<CancellationToken>,
|
execution_cancellation: Option<CancellationToken>,
|
||||||
disconnect: Option<NetworkRequestDisconnect>,
|
disconnect: Option<NetworkRequestDisconnect>,
|
||||||
|
cancellation: Option<NetworkRequestCancellation>,
|
||||||
decision_on_drop: PendingApprovalDecision,
|
decision_on_drop: PendingApprovalDecision,
|
||||||
completed: bool,
|
completed: bool,
|
||||||
}
|
}
|
||||||
@@ -297,6 +300,7 @@ impl<'a> PendingHostApprovalOwner<'a> {
|
|||||||
pending,
|
pending,
|
||||||
execution_cancellation,
|
execution_cancellation,
|
||||||
disconnect: None,
|
disconnect: None,
|
||||||
|
cancellation: None,
|
||||||
decision_on_drop: PendingApprovalDecision::Deny,
|
decision_on_drop: PendingApprovalDecision::Deny,
|
||||||
completed: false,
|
completed: false,
|
||||||
}
|
}
|
||||||
@@ -342,6 +346,17 @@ impl<'a> PendingHostApprovalOwner<'a> {
|
|||||||
impl Drop for PendingHostApprovalOwner<'_> {
|
impl Drop for PendingHostApprovalOwner<'_> {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
if !self.completed {
|
if !self.completed {
|
||||||
|
if self
|
||||||
|
.cancellation
|
||||||
|
.as_ref()
|
||||||
|
.and_then(NetworkRequestCancellation::reason)
|
||||||
|
== Some(NetworkRequestCancellationReason::ProcessFinished)
|
||||||
|
{
|
||||||
|
// Normal process cleanup is not a new review failure. Any earlier
|
||||||
|
// denial is already recorded in the call outcome.
|
||||||
|
self.publish_and_remove(self.decision_on_drop);
|
||||||
|
return;
|
||||||
|
}
|
||||||
if matches!(self.decision_on_drop, PendingApprovalDecision::Deny)
|
if matches!(self.decision_on_drop, PendingApprovalDecision::Deny)
|
||||||
&& let Some(registration_id) = self.key.execution_id.as_deref()
|
&& let Some(registration_id) = self.key.execution_id.as_deref()
|
||||||
&& let Some(elapsed) = self
|
&& let Some(elapsed) = self
|
||||||
@@ -517,20 +532,27 @@ impl NetworkApprovalService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn record_call_outcome(&self, registration_id: &str, outcome: String) {
|
fn record_call_outcome(&self, registration_id: &str, outcome: String) {
|
||||||
|
if let Some(cancellation_token) = self.store_call_outcome(registration_id, outcome) {
|
||||||
|
cancellation_token.cancel();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn store_call_outcome(
|
||||||
|
&self,
|
||||||
|
registration_id: &str,
|
||||||
|
outcome: String,
|
||||||
|
) -> Option<CancellationToken> {
|
||||||
let mut calls = self
|
let mut calls = self
|
||||||
.calls
|
.calls
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
let Some(call) = calls.active_calls.get(registration_id).cloned() else {
|
let call = calls.active_calls.get(registration_id)?.clone();
|
||||||
return;
|
|
||||||
};
|
|
||||||
// Explicit network-review outcomes replace generic blocked-request fallbacks.
|
// Explicit network-review outcomes replace generic blocked-request fallbacks.
|
||||||
calls
|
calls
|
||||||
.call_outcomes
|
.call_outcomes
|
||||||
.insert(registration_id.to_string(), outcome);
|
.insert(registration_id.to_string(), outcome);
|
||||||
|
|
||||||
drop(calls);
|
Some(call.cancellation_token.clone())
|
||||||
call.cancellation_token.cancel();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn record_call_outcome_if_absent(&self, registration_id: &str, outcome: String) {
|
fn record_call_outcome_if_absent(&self, registration_id: &str, outcome: String) {
|
||||||
@@ -676,6 +698,7 @@ impl NetworkApprovalService {
|
|||||||
.map(|call| call.cancellation_token.clone()),
|
.map(|call| call.cancellation_token.clone()),
|
||||||
);
|
);
|
||||||
pending_owner.disconnect = request.disconnect.clone();
|
pending_owner.disconnect = request.disconnect.clone();
|
||||||
|
pending_owner.cancellation = request.cancellation.clone();
|
||||||
|
|
||||||
let permission_profile = owner_call
|
let permission_profile = owner_call
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -814,6 +837,18 @@ impl NetworkApprovalService {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if matches!(
|
||||||
|
&approval_decision,
|
||||||
|
ReviewDecision::NetworkPolicyAmendment { network_policy_amendment }
|
||||||
|
if network_policy_amendment.action == NetworkPolicyRuleAction::Deny
|
||||||
|
) && let Some(owner_call) = owner_call.as_ref()
|
||||||
|
{
|
||||||
|
// Preserve the denial before waiting for the policy commit. Defer
|
||||||
|
// cancellation until after saving so it cannot interrupt persistence.
|
||||||
|
let _ = self
|
||||||
|
.store_call_outcome(&owner_call.registration_id, "rejected by user".to_string());
|
||||||
|
}
|
||||||
|
|
||||||
let _session_policy_commit_guard = if matches!(
|
let _session_policy_commit_guard = if matches!(
|
||||||
&approval_decision,
|
&approval_decision,
|
||||||
ReviewDecision::Approved
|
ReviewDecision::Approved
|
||||||
|
|||||||
@@ -0,0 +1,294 @@
|
|||||||
|
//! Exercises process cleanup while a remote network review is still pending.
|
||||||
|
|
||||||
|
use super::PushedExecScenario;
|
||||||
|
use super::accept_initialized_exec_server;
|
||||||
|
use super::read_exec_server_json;
|
||||||
|
use super::respond_environment_info;
|
||||||
|
use super::send_exec_server_json;
|
||||||
|
use anyhow::Context;
|
||||||
|
use anyhow::Result;
|
||||||
|
use base64::Engine;
|
||||||
|
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
||||||
|
use codex_core::TurnInputRequest;
|
||||||
|
use codex_protocol::config_types::ApprovalsReviewer;
|
||||||
|
use codex_protocol::items::CommandExecutionStatus;
|
||||||
|
use codex_protocol::items::TurnItem;
|
||||||
|
use codex_protocol::protocol::AskForApproval;
|
||||||
|
use codex_protocol::protocol::EventMsg;
|
||||||
|
use codex_protocol::protocol::ThreadSettingsOverrides;
|
||||||
|
use codex_protocol::user_input::UserInput;
|
||||||
|
use core_test_support::managed_network_requirements_loader;
|
||||||
|
use core_test_support::responses::ev_assistant_message;
|
||||||
|
use core_test_support::responses::ev_completed;
|
||||||
|
use core_test_support::responses::ev_function_call;
|
||||||
|
use core_test_support::responses::mount_response_once_match;
|
||||||
|
use core_test_support::responses::mount_sse_once_match;
|
||||||
|
use core_test_support::responses::sse;
|
||||||
|
use core_test_support::responses::sse_response;
|
||||||
|
use core_test_support::responses::start_mock_server;
|
||||||
|
use core_test_support::startup::STARTUP_TIMEOUT;
|
||||||
|
use core_test_support::test_codex::test_codex;
|
||||||
|
use core_test_support::test_codex::turn_permission_fields;
|
||||||
|
use pretty_assertions::assert_eq;
|
||||||
|
use serde_json::Value;
|
||||||
|
use serde_json::json;
|
||||||
|
use std::fs;
|
||||||
|
use std::time::Duration;
|
||||||
|
use tokio::net::TcpListener;
|
||||||
|
use tokio::net::TcpStream;
|
||||||
|
use tokio::time::timeout;
|
||||||
|
use tokio_tungstenite::WebSocketStream;
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn completed_remote_process_withdraws_pending_network_review() -> Result<()> {
|
||||||
|
let server = start_mock_server().await;
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await?;
|
||||||
|
let exec_server_url = format!("ws://{}", listener.local_addr()?);
|
||||||
|
let call_id = "pending-network-at-exit";
|
||||||
|
let parent = mount_sse_once_match(
|
||||||
|
&server,
|
||||||
|
|request: &wiremock::Request| {
|
||||||
|
let body = review_body(request);
|
||||||
|
body.pointer("/client_metadata/x-openai-subagent") != Some(&json!("guardian"))
|
||||||
|
&& !body.to_string().contains("pending-network-at-exit")
|
||||||
|
},
|
||||||
|
sse(vec![
|
||||||
|
ev_function_call(
|
||||||
|
call_id,
|
||||||
|
"exec_command",
|
||||||
|
&json!({"cmd": "build-site", "yield_time_ms": 10_000}).to_string(),
|
||||||
|
),
|
||||||
|
ev_completed("start"),
|
||||||
|
]),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let pending_review = mount_response_once_match(
|
||||||
|
&server,
|
||||||
|
|request: &wiremock::Request| {
|
||||||
|
review_body(request).pointer("/client_metadata/x-openai-subagent")
|
||||||
|
== Some(&json!("guardian"))
|
||||||
|
},
|
||||||
|
sse_response(sse(vec![ev_completed("review")]))
|
||||||
|
.set_delay(Duration::from_secs(/*secs*/ 60)),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let final_response = mount_sse_once_match(
|
||||||
|
&server,
|
||||||
|
|request: &wiremock::Request| {
|
||||||
|
let body = review_body(request);
|
||||||
|
body.pointer("/client_metadata/x-openai-subagent") != Some(&json!("guardian"))
|
||||||
|
&& body.to_string().contains("pending-network-at-exit")
|
||||||
|
},
|
||||||
|
sse(vec![
|
||||||
|
ev_assistant_message("done", "done"),
|
||||||
|
ev_completed("done"),
|
||||||
|
]),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let exec_server = tokio::spawn(async move {
|
||||||
|
let mut websocket = accept_initialized_exec_server(listener).await;
|
||||||
|
let start = loop {
|
||||||
|
let request = read_exec_server_json(&mut websocket, STARTUP_TIMEOUT).await;
|
||||||
|
if request["method"] == "process/start" {
|
||||||
|
break request;
|
||||||
|
}
|
||||||
|
respond_setup_request(&mut websocket, &request).await;
|
||||||
|
};
|
||||||
|
assert!(
|
||||||
|
start["params"]["networkProxy"]["policyDecisionTimeoutMs"]
|
||||||
|
.as_u64()
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
let process_id = &start["params"]["processId"];
|
||||||
|
send_exec_server_json(
|
||||||
|
&mut websocket,
|
||||||
|
json!({"id": start["id"], "result": {"processId": process_id}}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
send_exec_server_json(&mut websocket, json!({
|
||||||
|
"id": 1, "method": "network/policyRequest",
|
||||||
|
"params": {"processId": process_id, "request": {"protocol": "https_connect", "host": "review.example", "port": 443}}
|
||||||
|
})).await;
|
||||||
|
timeout(Duration::from_secs(/*secs*/ 10), async {
|
||||||
|
while !pending_review.requests().iter().any(|request| {
|
||||||
|
request
|
||||||
|
.body_json()
|
||||||
|
.pointer("/client_metadata/x-openai-subagent")
|
||||||
|
== Some(&json!("guardian"))
|
||||||
|
}) {
|
||||||
|
tokio::select! {
|
||||||
|
_ = tokio::time::sleep(Duration::from_millis(/*millis*/ 10)) => {},
|
||||||
|
request = read_exec_server_json(&mut websocket, Duration::from_secs(/*secs*/ 10)) => {
|
||||||
|
respond_setup_request(&mut websocket, &request).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("Guardian must start before process completion");
|
||||||
|
for message in [
|
||||||
|
json!({"method": "process/output", "params": {"processId": process_id, "seq": 1, "stream": "stdout", "chunk": BASE64_STANDARD.encode("build complete\n")}}),
|
||||||
|
json!({"method": "process/exited", "params": {"processId": process_id, "seq": 2, "exitCode": 0, "sandboxDenied": false}}),
|
||||||
|
json!({"method": "process/closed", "params": {"processId": process_id, "seq": 3}}),
|
||||||
|
] {
|
||||||
|
send_exec_server_json(&mut websocket, message).await;
|
||||||
|
}
|
||||||
|
loop {
|
||||||
|
let request =
|
||||||
|
read_exec_server_json(&mut websocket, Duration::from_secs(/*secs*/ 10)).await;
|
||||||
|
match request["method"].as_str() {
|
||||||
|
Some("process/terminate") => {
|
||||||
|
send_exec_server_json(
|
||||||
|
&mut websocket,
|
||||||
|
json!({"id": request["id"], "result": {"running": false}}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Some("process/read") => {
|
||||||
|
send_exec_server_json(&mut websocket, json!({"id": request["id"], "result": {"chunks": [], "nextSeq": 4, "exited": true, "exitCode": 0, "closed": true, "failure": null, "sandboxDenied": false}})).await;
|
||||||
|
}
|
||||||
|
None => panic!("withdrawn request must not be approved: {request}"),
|
||||||
|
_ => respond_setup_request(&mut websocket, &request).await,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let test = test_codex()
|
||||||
|
.with_exec_server_url(exec_server_url)
|
||||||
|
.with_cloud_config_bundle(managed_network_requirements_loader())
|
||||||
|
.with_pre_build_hook(|home| {
|
||||||
|
fs::write(
|
||||||
|
home.join("config.toml"),
|
||||||
|
r#"default_permissions = "workspace"
|
||||||
|
[permissions.workspace.filesystem]
|
||||||
|
":minimal" = "read"
|
||||||
|
[permissions.workspace.network]
|
||||||
|
enabled = true
|
||||||
|
mode = "full"
|
||||||
|
allow_local_binding = true
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.expect("managed network config");
|
||||||
|
})
|
||||||
|
.with_config(|config| {
|
||||||
|
config.project_doc_max_bytes = 0;
|
||||||
|
config.approvals_reviewer = ApprovalsReviewer::AutoReview;
|
||||||
|
#[cfg(windows)]
|
||||||
|
config.set_windows_sandbox_enabled(/*value*/ true);
|
||||||
|
})
|
||||||
|
// This test supplies its own fake executor and must not select a CI executor.
|
||||||
|
.build(&server)
|
||||||
|
.await?;
|
||||||
|
let (sandbox_policy, permission_profile) = turn_permission_fields(
|
||||||
|
test.session_configured.permission_profile.clone(),
|
||||||
|
test.config.cwd.as_path(),
|
||||||
|
);
|
||||||
|
test.codex
|
||||||
|
.start_or_steer_turn(
|
||||||
|
TurnInputRequest::user_input(vec![UserInput::Text {
|
||||||
|
text: "build the site".into(),
|
||||||
|
text_elements: Vec::new(),
|
||||||
|
}])
|
||||||
|
.with_thread_settings(ThreadSettingsOverrides {
|
||||||
|
approval_policy: Some(AskForApproval::OnRequest),
|
||||||
|
approvals_reviewer: Some(ApprovalsReviewer::AutoReview),
|
||||||
|
sandbox_policy: Some(sandbox_policy),
|
||||||
|
permission_profile,
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let completed = timeout(Duration::from_secs(/*secs*/ 15), async {
|
||||||
|
loop {
|
||||||
|
match test.codex.next_event().await?.msg {
|
||||||
|
EventMsg::ItemCompleted(event) => {
|
||||||
|
if let TurnItem::CommandExecution(item) = event.item {
|
||||||
|
return Ok::<_, anyhow::Error>(item);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
EventMsg::ExecApprovalRequest(_) => anyhow::bail!("unexpected user approval"),
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.context("command should complete")??;
|
||||||
|
assert_eq!(
|
||||||
|
(
|
||||||
|
completed.status,
|
||||||
|
completed.exit_code,
|
||||||
|
completed.aggregated_output.as_deref()
|
||||||
|
),
|
||||||
|
(
|
||||||
|
CommandExecutionStatus::Completed,
|
||||||
|
Some(0),
|
||||||
|
Some("build complete\n")
|
||||||
|
)
|
||||||
|
);
|
||||||
|
timeout(Duration::from_secs(/*secs*/ 10), async {
|
||||||
|
while final_response.function_call_output_text(call_id).is_none() {
|
||||||
|
tokio::time::sleep(Duration::from_millis(/*millis*/ 10)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
let output = final_response
|
||||||
|
.function_call_output_text(call_id)
|
||||||
|
.context("tool output")?;
|
||||||
|
assert!(output.contains("Process exited with code 0"), "{output}");
|
||||||
|
assert!(output.contains("build complete"), "{output}");
|
||||||
|
assert_eq!(parent.requests().len(), 1);
|
||||||
|
timeout(Duration::from_secs(/*secs*/ 10), exec_server).await??;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn review_body(request: &wiremock::Request) -> Value {
|
||||||
|
let bytes = if request
|
||||||
|
.headers
|
||||||
|
.get("content-encoding")
|
||||||
|
.is_some_and(|value| value == "zstd")
|
||||||
|
{
|
||||||
|
zstd::stream::decode_all(std::io::Cursor::new(&request.body)).expect("decode request")
|
||||||
|
} else {
|
||||||
|
request.body.clone()
|
||||||
|
};
|
||||||
|
serde_json::from_slice(&bytes).expect("request JSON")
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn respond_setup_request(websocket: &mut WebSocketStream<TcpStream>, request: &Value) {
|
||||||
|
match request["method"].as_str() {
|
||||||
|
Some("environment/info") => {
|
||||||
|
respond_environment_info(websocket, &request["id"], PushedExecScenario::Complete).await;
|
||||||
|
}
|
||||||
|
Some("fs/getMetadata" | "fs/readFile" | "fs/readDirectory") => {
|
||||||
|
send_exec_server_json(
|
||||||
|
websocket,
|
||||||
|
json!({
|
||||||
|
"id": request["id"],
|
||||||
|
"error": {"code": -32004, "message": "not found"}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
Some("fs/canonicalize") => {
|
||||||
|
send_exec_server_json(
|
||||||
|
websocket,
|
||||||
|
json!({
|
||||||
|
"id": request["id"],
|
||||||
|
"result": {"path": request["params"]["path"]}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
Some("fs/walk") => {
|
||||||
|
send_exec_server_json(
|
||||||
|
websocket,
|
||||||
|
json!({
|
||||||
|
"id": request["id"],
|
||||||
|
"result": {"entries": [], "errors": [], "truncated": false}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
method => panic!("unexpected setup request: {method:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1075,3 +1075,6 @@ timeout = 900
|
|||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[path = "network_approval_completion.rs"]
|
||||||
|
mod network_approval_completion;
|
||||||
@@ -13,6 +13,8 @@ use arc_swap::ArcSwapOption;
|
|||||||
use codex_exec_server_protocol::JSONRPCNotification;
|
use codex_exec_server_protocol::JSONRPCNotification;
|
||||||
use codex_network_proxy::NetworkPolicyDecider;
|
use codex_network_proxy::NetworkPolicyDecider;
|
||||||
use codex_network_proxy::NetworkProxyAuditMetadata;
|
use codex_network_proxy::NetworkProxyAuditMetadata;
|
||||||
|
use codex_network_proxy::NetworkRequestCancellation;
|
||||||
|
use codex_network_proxy::NetworkRequestCancellationReason;
|
||||||
use futures::FutureExt;
|
use futures::FutureExt;
|
||||||
use futures::future::BoxFuture;
|
use futures::future::BoxFuture;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
@@ -211,6 +213,7 @@ pub(crate) struct SessionState {
|
|||||||
struct NetworkPolicyState {
|
struct NetworkPolicyState {
|
||||||
controller: ArcSwapOption<NetworkPolicyDecisionController>,
|
controller: ArcSwapOption<NetworkPolicyDecisionController>,
|
||||||
cancelled: CancellationToken,
|
cancelled: CancellationToken,
|
||||||
|
cancellation: NetworkRequestCancellation,
|
||||||
audit: Option<NetworkPolicyAuditContext>,
|
audit: Option<NetworkPolicyAuditContext>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -877,6 +880,13 @@ impl ExecServerClient {
|
|||||||
&self,
|
&self,
|
||||||
process_id: &ProcessId,
|
process_id: &ProcessId,
|
||||||
) -> Result<TerminateResponse, ExecServerError> {
|
) -> Result<TerminateResponse, ExecServerError> {
|
||||||
|
// A close notification may arrive before the termination response.
|
||||||
|
if let Some(session) = self.inner.get_session(process_id) {
|
||||||
|
session
|
||||||
|
.network_policy
|
||||||
|
.cancellation
|
||||||
|
.record(NetworkRequestCancellationReason::ProcessCancelled);
|
||||||
|
}
|
||||||
self.call_for_cleanup(
|
self.call_for_cleanup(
|
||||||
EXEC_TERMINATE_METHOD,
|
EXEC_TERMINATE_METHOD,
|
||||||
&TerminateParams {
|
&TerminateParams {
|
||||||
@@ -1318,6 +1328,7 @@ impl SessionState {
|
|||||||
network_policy: NetworkPolicyState {
|
network_policy: NetworkPolicyState {
|
||||||
controller: ArcSwapOption::empty(),
|
controller: ArcSwapOption::empty(),
|
||||||
cancelled: CancellationToken::new(),
|
cancelled: CancellationToken::new(),
|
||||||
|
cancellation: NetworkRequestCancellation::default(),
|
||||||
audit: None,
|
audit: None,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -1381,6 +1392,11 @@ impl SessionState {
|
|||||||
let is_closed = matches!(&event, ExecProcessEvent::Closed { .. });
|
let is_closed = matches!(&event, ExecProcessEvent::Closed { .. });
|
||||||
ordered_events.closed_published |= is_closed;
|
ordered_events.closed_published |= is_closed;
|
||||||
published_closed |= is_closed;
|
published_closed |= is_closed;
|
||||||
|
if is_closed && ordered_events.exit_published {
|
||||||
|
self.network_policy
|
||||||
|
.cancellation
|
||||||
|
.record(NetworkRequestCancellationReason::ProcessFinished);
|
||||||
|
}
|
||||||
self.events.publish(event);
|
self.events.publish(event);
|
||||||
}
|
}
|
||||||
published_closed
|
published_closed
|
||||||
@@ -1581,6 +1597,10 @@ impl Session {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn cancel_network_policy_decisions(&self) {
|
pub(crate) fn cancel_network_policy_decisions(&self) {
|
||||||
|
self.state
|
||||||
|
.network_policy
|
||||||
|
.cancellation
|
||||||
|
.record(NetworkRequestCancellationReason::ProcessCancelled);
|
||||||
self.state.network_policy.cancelled.cancel();
|
self.state.network_policy.cancelled.cancel();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1638,6 +1658,10 @@ impl Inner {
|
|||||||
let mut next_sessions = sessions.as_ref().clone();
|
let mut next_sessions = sessions.as_ref().clone();
|
||||||
next_sessions.remove(process_id);
|
next_sessions.remove(process_id);
|
||||||
self.sessions.store(Arc::new(next_sessions));
|
self.sessions.store(Arc::new(next_sessions));
|
||||||
|
expected
|
||||||
|
.network_policy
|
||||||
|
.cancellation
|
||||||
|
.record(NetworkRequestCancellationReason::ProcessCancelled);
|
||||||
expected.network_policy.cancelled.cancel();
|
expected.network_policy.cancelled.cancel();
|
||||||
expected.network_policy.controller.store(None);
|
expected.network_policy.controller.store(None);
|
||||||
}
|
}
|
||||||
@@ -1678,6 +1702,10 @@ fn fail_all_sessions(inner: &Arc<Inner>, message: String) {
|
|||||||
let sessions = inner.take_all_sessions();
|
let sessions = inner.take_all_sessions();
|
||||||
|
|
||||||
for (_, session) in sessions {
|
for (_, session) in sessions {
|
||||||
|
session
|
||||||
|
.network_policy
|
||||||
|
.cancellation
|
||||||
|
.record(NetworkRequestCancellationReason::ConnectionClosed);
|
||||||
session.network_policy.cancelled.cancel();
|
session.network_policy.cancelled.cancel();
|
||||||
session.network_policy.controller.store(None);
|
session.network_policy.controller.store(None);
|
||||||
// Sessions synthesize a closed read response and emit a pushed Failed
|
// Sessions synthesize a closed read response and emit a pushed Failed
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ use codex_network_proxy::NetworkDecision;
|
|||||||
use codex_network_proxy::NetworkPolicyDecision;
|
use codex_network_proxy::NetworkPolicyDecision;
|
||||||
use codex_network_proxy::NetworkPolicyRequest;
|
use codex_network_proxy::NetworkPolicyRequest;
|
||||||
use codex_network_proxy::NetworkProtocol;
|
use codex_network_proxy::NetworkProtocol;
|
||||||
|
use codex_network_proxy::NetworkRequestCancellation;
|
||||||
|
use codex_network_proxy::NetworkRequestCancellationReason;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
use tokio::time::Instant;
|
use tokio::time::Instant;
|
||||||
@@ -526,6 +528,10 @@ impl Inner {
|
|||||||
Ok(true) => self.remove_session_if(process_id, session),
|
Ok(true) => self.remove_session_if(process_id, session),
|
||||||
Ok(false) => {}
|
Ok(false) => {}
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
|
session
|
||||||
|
.network_policy
|
||||||
|
.cancellation
|
||||||
|
.record(NetworkRequestCancellationReason::ProcessCancelled);
|
||||||
let terminated: Result<TerminateResponse, ExecServerError> = rpc_client
|
let terminated: Result<TerminateResponse, ExecServerError> = rpc_client
|
||||||
.call_for_cleanup(
|
.call_for_cleanup(
|
||||||
EXEC_TERMINATE_METHOD,
|
EXEC_TERMINATE_METHOD,
|
||||||
@@ -688,6 +694,10 @@ impl ExecServerClient {
|
|||||||
let process_cancelled = session
|
let process_cancelled = session
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|session| session.network_policy.cancelled.clone());
|
.map(|session| session.network_policy.cancelled.clone());
|
||||||
|
let process_cancellation = session
|
||||||
|
.as_ref()
|
||||||
|
.map(|session| session.network_policy.cancellation.clone());
|
||||||
|
let cancellation = NetworkRequestCancellation::default();
|
||||||
let expected_session = session.as_ref().map(Arc::downgrade);
|
let expected_session = session.as_ref().map(Arc::downgrade);
|
||||||
let policy_request =
|
let policy_request =
|
||||||
(process_id_valid && host_valid).then_some(NetworkPolicyRequest {
|
(process_id_valid && host_valid).then_some(NetworkPolicyRequest {
|
||||||
@@ -712,6 +722,7 @@ impl ExecServerClient {
|
|||||||
exec_policy_hint: None,
|
exec_policy_hint: None,
|
||||||
execution_id: None,
|
execution_id: None,
|
||||||
disconnect: None,
|
disconnect: None,
|
||||||
|
cancellation: Some(cancellation.clone()),
|
||||||
});
|
});
|
||||||
let inner = Arc::downgrade(&inner);
|
let inner = Arc::downgrade(&inner);
|
||||||
let rpc_client = Arc::downgrade(&rpc_client);
|
let rpc_client = Arc::downgrade(&rpc_client);
|
||||||
@@ -721,18 +732,26 @@ impl ExecServerClient {
|
|||||||
let _request_guard = request_guard;
|
let _request_guard = request_guard;
|
||||||
let decision = match (controller, policy_request, process_cancelled) {
|
let decision = match (controller, policy_request, process_cancelled) {
|
||||||
(Some(controller), Some(request), Some(process_cancelled)) => {
|
(Some(controller), Some(request), Some(process_cancelled)) => {
|
||||||
// Core's pending-approval guard makes dropping this
|
// Keep the decision future outside select/timeout so its
|
||||||
// future on process removal or deadline fail closed.
|
// guard sees the cancellation cause before it is dropped.
|
||||||
|
let mut decision = controller.decider.decide(request);
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
biased;
|
biased;
|
||||||
_ = connection_cancelled.cancelled() => return,
|
_ = connection_cancelled.cancelled() => {
|
||||||
|
cancellation.record(NetworkRequestCancellationReason::ConnectionClosed);
|
||||||
|
return;
|
||||||
|
},
|
||||||
_ = process_cancelled.cancelled() => {
|
_ = process_cancelled.cancelled() => {
|
||||||
|
cancellation.record(process_cancellation.as_ref()
|
||||||
|
.and_then(NetworkRequestCancellation::reason)
|
||||||
|
.unwrap_or(NetworkRequestCancellationReason::ProcessCancelled));
|
||||||
NetworkDecision::deny(NETWORK_POLICY_DENIAL_REASON)
|
NetworkDecision::deny(NETWORK_POLICY_DENIAL_REASON)
|
||||||
}
|
}
|
||||||
decision = timeout(
|
result = timeout(
|
||||||
controller.timeout,
|
controller.timeout,
|
||||||
controller.decider.decide(request),
|
&mut decision,
|
||||||
) => decision.unwrap_or_else(|_| {
|
) => result.unwrap_or_else(|_| {
|
||||||
|
cancellation.record(NetworkRequestCancellationReason::TimedOut);
|
||||||
NetworkDecision::deny(NETWORK_POLICY_DENIAL_REASON)
|
NetworkDecision::deny(NETWORK_POLICY_DENIAL_REASON)
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ mod process_log_metadata;
|
|||||||
mod proxy;
|
mod proxy;
|
||||||
mod reasons;
|
mod reasons;
|
||||||
mod remote_config;
|
mod remote_config;
|
||||||
|
mod request_cancellation;
|
||||||
mod request_disconnect;
|
mod request_disconnect;
|
||||||
mod responses;
|
mod responses;
|
||||||
mod runtime;
|
mod runtime;
|
||||||
@@ -89,6 +90,8 @@ pub use proxy::proxy_url_env_value;
|
|||||||
pub use proxy::strip_managed_proxy_env;
|
pub use proxy::strip_managed_proxy_env;
|
||||||
pub use remote_config::RemoteNetworkProxyConfig;
|
pub use remote_config::RemoteNetworkProxyConfig;
|
||||||
pub use remote_config::RemoteNetworkProxyLaunchConfig;
|
pub use remote_config::RemoteNetworkProxyLaunchConfig;
|
||||||
|
pub use request_cancellation::NetworkRequestCancellation;
|
||||||
|
pub use request_cancellation::NetworkRequestCancellationReason;
|
||||||
pub use request_disconnect::NetworkRequestDisconnect;
|
pub use request_disconnect::NetworkRequestDisconnect;
|
||||||
pub use runtime::BlockedRequest;
|
pub use runtime::BlockedRequest;
|
||||||
pub use runtime::BlockedRequestArgs;
|
pub use runtime::BlockedRequestArgs;
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
use crate::reasons::REASON_POLICY_DENIED;
|
use crate::reasons::REASON_POLICY_DENIED;
|
||||||
|
use crate::request_cancellation::NetworkRequestCancellation;
|
||||||
use crate::request_disconnect::NetworkRequestDisconnect;
|
use crate::request_disconnect::NetworkRequestDisconnect;
|
||||||
use crate::runtime::HostBlockDecision;
|
use crate::runtime::HostBlockDecision;
|
||||||
use crate::runtime::HostBlockReason;
|
use crate::runtime::HostBlockReason;
|
||||||
@@ -109,6 +110,8 @@ pub struct NetworkPolicyRequest {
|
|||||||
pub execution_id: Option<String>,
|
pub execution_id: Option<String>,
|
||||||
/// Present only when the local HTTP transport can identify an abandoned request.
|
/// Present only when the local HTTP transport can identify an abandoned request.
|
||||||
pub disconnect: Option<NetworkRequestDisconnect>,
|
pub disconnect: Option<NetworkRequestDisconnect>,
|
||||||
|
/// Controller-owned cause, published before an abandoned decision future is dropped.
|
||||||
|
pub cancellation: Option<NetworkRequestCancellation>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct NetworkPolicyRequestArgs {
|
pub struct NetworkPolicyRequestArgs {
|
||||||
@@ -145,6 +148,7 @@ impl NetworkPolicyRequest {
|
|||||||
exec_policy_hint,
|
exec_policy_hint,
|
||||||
execution_id: None,
|
execution_id: None,
|
||||||
disconnect: None,
|
disconnect: None,
|
||||||
|
cancellation: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
//! Records why a policy request was withdrawn before its decision future is dropped.
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::OnceLock;
|
||||||
|
|
||||||
|
/// The controller's first known reason for withdrawing a policy request.
|
||||||
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
|
pub enum NetworkRequestCancellationReason {
|
||||||
|
/// The process exited and its output streams closed normally.
|
||||||
|
ProcessFinished,
|
||||||
|
/// The process was explicitly terminated or its handle was abandoned.
|
||||||
|
ProcessCancelled,
|
||||||
|
/// The connection to the executor was lost.
|
||||||
|
ConnectionClosed,
|
||||||
|
/// The policy decision deadline expired.
|
||||||
|
TimedOut,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Shared, in-process metadata; recording a reason does not authorize network access.
|
||||||
|
#[derive(Clone, Debug, Default)]
|
||||||
|
pub struct NetworkRequestCancellation(Arc<OnceLock<NetworkRequestCancellationReason>>);
|
||||||
|
|
||||||
|
impl NetworkRequestCancellation {
|
||||||
|
pub fn reason(&self) -> Option<NetworkRequestCancellationReason> {
|
||||||
|
self.0.get().copied()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Publish before dropping the decision future. Cleanup cannot replace an earlier cause.
|
||||||
|
pub fn record(&self, reason: NetworkRequestCancellationReason) {
|
||||||
|
let _ = self.0.set(reason);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user