mirror of
https://github.com/openai/codex.git
synced 2026-11-16 06:54:36 +00:00
Let Guardian V2 satisfy required model reviews (#39658)
## What changed - Allow the Guardian V2 approval monitor to handle reviews for models that require automatic review. - Preserve full Guardian review when Guardian V2 is disabled by configuration or managed requirements. - Cover required-model routing for low- and high-risk actions, plus both Guardian V2 disable paths. GitOrigin-RevId: d1201cc3dbd7c4a07de633388b14f17ada9feb39
This commit is contained in:
1 parent
5e3a6fe4ee
commit
4e1a772a7d
3 files changed
+233
-12
No files matched your search
@@ -45,6 +45,7 @@ use super::mcp_tool::TEST_TOOL_NAME;
|
|||||||
use super::mcp_tool::start_mcp_server;
|
use super::mcp_tool::start_mcp_server;
|
||||||
|
|
||||||
const TIMEOUT: Duration = Duration::from_secs(30);
|
const TIMEOUT: Duration = Duration::from_secs(30);
|
||||||
|
const MODEL: &str = "mock-model";
|
||||||
const USER_CONTEXT: &str = "The user authorized reading the existing project files.";
|
const USER_CONTEXT: &str = "The user authorized reading the existing project files.";
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
@@ -65,6 +66,12 @@ enum GuardianRisk {
|
|||||||
High,
|
High,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
enum ModelReviewRequirement {
|
||||||
|
Optional,
|
||||||
|
Required,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
enum ThreadLifecycle {
|
enum ThreadLifecycle {
|
||||||
New,
|
New,
|
||||||
@@ -166,6 +173,7 @@ async fn luna_websocket(
|
|||||||
async fn guardian_v2_routes_tool_approvals(
|
async fn guardian_v2_routes_tool_approvals(
|
||||||
risk: GuardianRisk,
|
risk: GuardianRisk,
|
||||||
lifecycle: ThreadLifecycle,
|
lifecycle: ThreadLifecycle,
|
||||||
|
requirement: ModelReviewRequirement,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let (luna_score, expected_guardian_reviews) = match risk {
|
let (luna_score, expected_guardian_reviews) = match risk {
|
||||||
GuardianRisk::Low => (0.25, 1),
|
GuardianRisk::Low => (0.25, 1),
|
||||||
@@ -197,10 +205,24 @@ async fn guardian_v2_routes_tool_approvals(
|
|||||||
let (mcp_server_url, mcp_server_handle) = start_mcp_server().await?;
|
let (mcp_server_url, mcp_server_handle) = start_mcp_server().await?;
|
||||||
|
|
||||||
let codex_home = TempDir::new()?;
|
let codex_home = TempDir::new()?;
|
||||||
|
let (reviewer_config, requested_reviewer) = match requirement {
|
||||||
|
ModelReviewRequirement::Optional => (
|
||||||
|
"approvals_reviewer = \"auto_review\"",
|
||||||
|
ApprovalsReviewer::AutoReview,
|
||||||
|
),
|
||||||
|
ModelReviewRequirement::Required => {
|
||||||
|
std::fs::write(
|
||||||
|
codex_home.path().join("requirements.toml"),
|
||||||
|
format!("[auto_review]\nrequired_on_models = [\"{MODEL}\"]\n"),
|
||||||
|
)?;
|
||||||
|
("approvals_reviewer = \"user\"", ApprovalsReviewer::User)
|
||||||
|
}
|
||||||
|
};
|
||||||
MockResponsesConfig::new(&responses_url)
|
MockResponsesConfig::new(&responses_url)
|
||||||
|
.with_model(MODEL)
|
||||||
.with_provider_config("supports_websockets = false")
|
.with_provider_config("supports_websockets = false")
|
||||||
.with_approval_policy("on-request")
|
.with_approval_policy("on-request")
|
||||||
.with_root_config("approvals_reviewer = \"auto_review\"")
|
.with_root_config(reviewer_config)
|
||||||
.with_extra_config(&format!(
|
.with_extra_config(&format!(
|
||||||
"[mcp_servers.{TEST_SERVER_NAME}]\nurl = \"{mcp_server_url}/mcp\"\ndefault_tools_approval_mode = \"prompt\"\n\n[analytics]\nenabled = true\n\n[otel]\nmetrics_exporter = {{ otlp-http = {{ endpoint = \"{responses_url}/metrics\", protocol = \"json\" }} }}"
|
"[mcp_servers.{TEST_SERVER_NAME}]\nurl = \"{mcp_server_url}/mcp\"\ndefault_tools_approval_mode = \"prompt\"\n\n[analytics]\nenabled = true\n\n[otel]\nmetrics_exporter = {{ otlp-http = {{ endpoint = \"{responses_url}/metrics\", protocol = \"json\" }} }}"
|
||||||
))
|
))
|
||||||
@@ -225,14 +247,18 @@ async fn guardian_v2_routes_tool_approvals(
|
|||||||
.await?;
|
.await?;
|
||||||
let thread = match lifecycle {
|
let thread = match lifecycle {
|
||||||
ThreadLifecycle::New => {
|
ThreadLifecycle::New => {
|
||||||
app_server
|
let started = app_server
|
||||||
.start_thread(ThreadStartParams {
|
.start_thread(ThreadStartParams {
|
||||||
approval_policy: Some(AskForApproval::OnRequest),
|
approval_policy: Some(AskForApproval::OnRequest),
|
||||||
approvals_reviewer: Some(ApprovalsReviewer::AutoReview),
|
approvals_reviewer: Some(requested_reviewer),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
})
|
})
|
||||||
.await?
|
.await?;
|
||||||
.thread
|
assert_eq!(
|
||||||
|
(started.model.as_str(), started.approvals_reviewer),
|
||||||
|
(MODEL, ApprovalsReviewer::AutoReview)
|
||||||
|
);
|
||||||
|
started.thread
|
||||||
}
|
}
|
||||||
ThreadLifecycle::Resume => {
|
ThreadLifecycle::Resume => {
|
||||||
let original_thread_id = original_thread_id.expect("resumed thread should exist");
|
let original_thread_id = original_thread_id.expect("resumed thread should exist");
|
||||||
@@ -274,7 +300,10 @@ async fn guardian_v2_routes_tool_approvals(
|
|||||||
text_elements: Vec::new(),
|
text_elements: Vec::new(),
|
||||||
}],
|
}],
|
||||||
approval_policy: Some(AskForApproval::OnRequest),
|
approval_policy: Some(AskForApproval::OnRequest),
|
||||||
approvals_reviewer: Some(ApprovalsReviewer::AutoReview),
|
approvals_reviewer: match requirement {
|
||||||
|
ModelReviewRequirement::Optional => Some(ApprovalsReviewer::AutoReview),
|
||||||
|
ModelReviewRequirement::Required => None,
|
||||||
|
},
|
||||||
..Default::default()
|
..Default::default()
|
||||||
})
|
})
|
||||||
.await?;
|
.await?;
|
||||||
@@ -372,29 +401,76 @@ async fn guardian_v2_routes_tool_approvals(
|
|||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
async fn guardian_v2_low_risk_actions_skip_subsequent_reviews() -> Result<()> {
|
async fn guardian_v2_low_risk_actions_skip_subsequent_reviews() -> Result<()> {
|
||||||
skip_if_no_network!(Ok(()));
|
skip_if_no_network!(Ok(()));
|
||||||
guardian_v2_routes_tool_approvals(GuardianRisk::Low, ThreadLifecycle::New).await
|
guardian_v2_routes_tool_approvals(
|
||||||
|
GuardianRisk::Low,
|
||||||
|
ThreadLifecycle::New,
|
||||||
|
ModelReviewRequirement::Optional,
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
async fn guardian_v2_high_risk_actions_require_full_reviews() -> Result<()> {
|
async fn guardian_v2_high_risk_actions_require_full_reviews() -> Result<()> {
|
||||||
skip_if_no_network!(Ok(()));
|
skip_if_no_network!(Ok(()));
|
||||||
guardian_v2_routes_tool_approvals(GuardianRisk::High, ThreadLifecycle::New).await
|
guardian_v2_routes_tool_approvals(
|
||||||
|
GuardianRisk::High,
|
||||||
|
ThreadLifecycle::New,
|
||||||
|
ModelReviewRequirement::Optional,
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
async fn guardian_v2_threshold_score_requires_full_reviews() -> Result<()> {
|
async fn guardian_v2_threshold_score_requires_full_reviews() -> Result<()> {
|
||||||
skip_if_no_network!(Ok(()));
|
skip_if_no_network!(Ok(()));
|
||||||
guardian_v2_routes_tool_approvals(GuardianRisk::Threshold, ThreadLifecycle::New).await
|
guardian_v2_routes_tool_approvals(
|
||||||
|
GuardianRisk::Threshold,
|
||||||
|
ThreadLifecycle::New,
|
||||||
|
ModelReviewRequirement::Optional,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn guardian_v2_required_model_honors_low_risk_assessment() -> Result<()> {
|
||||||
|
skip_if_no_network!(Ok(()));
|
||||||
|
guardian_v2_routes_tool_approvals(
|
||||||
|
GuardianRisk::Low,
|
||||||
|
ThreadLifecycle::New,
|
||||||
|
ModelReviewRequirement::Required,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn guardian_v2_required_model_high_risk_requires_full_review() -> Result<()> {
|
||||||
|
skip_if_no_network!(Ok(()));
|
||||||
|
guardian_v2_routes_tool_approvals(
|
||||||
|
GuardianRisk::High,
|
||||||
|
ThreadLifecycle::New,
|
||||||
|
ModelReviewRequirement::Required,
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
async fn resumed_thread_starts_without_guardian_score() -> Result<()> {
|
async fn resumed_thread_starts_without_guardian_score() -> Result<()> {
|
||||||
skip_if_no_network!(Ok(()));
|
skip_if_no_network!(Ok(()));
|
||||||
guardian_v2_routes_tool_approvals(GuardianRisk::Low, ThreadLifecycle::Resume).await
|
guardian_v2_routes_tool_approvals(
|
||||||
|
GuardianRisk::Low,
|
||||||
|
ThreadLifecycle::Resume,
|
||||||
|
ModelReviewRequirement::Optional,
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
async fn forked_thread_starts_without_guardian_score() -> Result<()> {
|
async fn forked_thread_starts_without_guardian_score() -> Result<()> {
|
||||||
skip_if_no_network!(Ok(()));
|
skip_if_no_network!(Ok(()));
|
||||||
guardian_v2_routes_tool_approvals(GuardianRisk::Low, ThreadLifecycle::Fork).await
|
guardian_v2_routes_tool_approvals(
|
||||||
|
GuardianRisk::Low,
|
||||||
|
ThreadLifecycle::Fork,
|
||||||
|
ModelReviewRequirement::Optional,
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
@@ -313,11 +313,13 @@ async fn run_guardian_review(
|
|||||||
options: GuardianReviewOptions,
|
options: GuardianReviewOptions,
|
||||||
) -> ReviewDecision {
|
) -> ReviewDecision {
|
||||||
let turn = Arc::clone(context.turn());
|
let turn = Arc::clone(context.turn());
|
||||||
if !turn
|
// Required models must use Guardian, but an enabled V2 monitor can satisfy the review.
|
||||||
|
if (!turn
|
||||||
.config
|
.config
|
||||||
.config_layer_stack
|
.config_layer_stack
|
||||||
.requirements()
|
.requirements()
|
||||||
.auto_review_required_for_model(&turn.model_info.slug)
|
.auto_review_required_for_model(&turn.model_info.slug)
|
||||||
|
|| turn.config.features.enabled(Feature::GuardianV2))
|
||||||
&& reasons.retry.is_none()
|
&& reasons.retry.is_none()
|
||||||
&& options
|
&& options
|
||||||
.external_cancel
|
.external_cancel
|
||||||
|
|||||||
@@ -2,7 +2,10 @@ use codex_core::TurnInputRequest;
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use anyhow::Context;
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
use codex_config::test_support::CloudConfigBundleFixture;
|
||||||
|
use codex_core::config::Constrained;
|
||||||
use codex_extension_api::ApprovalReviewContributor;
|
use codex_extension_api::ApprovalReviewContributor;
|
||||||
use codex_extension_api::ExtensionData;
|
use codex_extension_api::ExtensionData;
|
||||||
use codex_extension_api::ExtensionFuture;
|
use codex_extension_api::ExtensionFuture;
|
||||||
@@ -29,6 +32,7 @@ use codex_protocol::protocol::Op;
|
|||||||
use codex_protocol::protocol::ReviewDecision;
|
use codex_protocol::protocol::ReviewDecision;
|
||||||
use codex_protocol::protocol::ThreadSettingsOverrides;
|
use codex_protocol::protocol::ThreadSettingsOverrides;
|
||||||
use codex_protocol::request_permissions::PermissionGrantScope;
|
use codex_protocol::request_permissions::PermissionGrantScope;
|
||||||
|
use codex_protocol::request_permissions::RequestPermissionProfile;
|
||||||
use codex_protocol::request_permissions::RequestPermissionsResponse;
|
use codex_protocol::request_permissions::RequestPermissionsResponse;
|
||||||
use codex_protocol::user_input::UserInput;
|
use codex_protocol::user_input::UserInput;
|
||||||
use core_test_support::TempDirExt;
|
use core_test_support::TempDirExt;
|
||||||
@@ -50,6 +54,7 @@ use core_test_support::wait_for_event;
|
|||||||
use core_test_support::wait_for_event_with_timeout;
|
use core_test_support::wait_for_event_with_timeout;
|
||||||
use pretty_assertions::assert_eq;
|
use pretty_assertions::assert_eq;
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
use test_case::test_case;
|
||||||
use tokio::time::timeout;
|
use tokio::time::timeout;
|
||||||
use wiremock::Mock;
|
use wiremock::Mock;
|
||||||
use wiremock::MockServer;
|
use wiremock::MockServer;
|
||||||
@@ -155,6 +160,144 @@ async fn approval_review_contributor_skips_existing_guardian_model_call() -> Res
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
enum GuardianV2DisableSource {
|
||||||
|
Config,
|
||||||
|
ManagedRequirements,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test_case(GuardianV2DisableSource::Config; "disabled in config")]
|
||||||
|
#[test_case(GuardianV2DisableSource::ManagedRequirements; "disabled by managed policy")]
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn required_model_bypasses_extension_approval_when_guardian_v2_is_disabled(
|
||||||
|
disable_source: GuardianV2DisableSource,
|
||||||
|
) -> Result<()> {
|
||||||
|
skip_if_no_network!(Ok(()));
|
||||||
|
skip_if_wine_exec!(Ok(()), "request_permissions requires host-native paths");
|
||||||
|
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
let model = "gpt-5.4";
|
||||||
|
let call_id = "required-model-permissions";
|
||||||
|
let reason = "request network access under mandatory Guardian review";
|
||||||
|
let responses = mount_sse_sequence(
|
||||||
|
&server,
|
||||||
|
vec![
|
||||||
|
sse(vec![
|
||||||
|
ev_response_created("resp-required-model-parent"),
|
||||||
|
ev_function_call(
|
||||||
|
call_id,
|
||||||
|
"request_permissions",
|
||||||
|
&json!({
|
||||||
|
"reason": reason,
|
||||||
|
"permissions": { "network": { "enabled": true } },
|
||||||
|
})
|
||||||
|
.to_string(),
|
||||||
|
),
|
||||||
|
ev_completed("resp-required-model-parent"),
|
||||||
|
]),
|
||||||
|
sse(vec![
|
||||||
|
ev_response_created("resp-required-model-review"),
|
||||||
|
ev_assistant_message(
|
||||||
|
"msg-required-model-review",
|
||||||
|
&json!({
|
||||||
|
"risk_level": "high",
|
||||||
|
"user_authorization": "low",
|
||||||
|
"outcome": "deny",
|
||||||
|
"rationale": "The requested network access is not authorized.",
|
||||||
|
})
|
||||||
|
.to_string(),
|
||||||
|
),
|
||||||
|
ev_completed("resp-required-model-review"),
|
||||||
|
]),
|
||||||
|
sse(vec![
|
||||||
|
ev_response_created("resp-required-model-followup"),
|
||||||
|
ev_assistant_message("msg-required-model-followup", "done"),
|
||||||
|
ev_completed("resp-required-model-followup"),
|
||||||
|
]),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let (guardian_v2_config, reviewer_requirements) = match disable_source {
|
||||||
|
GuardianV2DisableSource::Config => ("false", ""),
|
||||||
|
GuardianV2DisableSource::ManagedRequirements => {
|
||||||
|
("true", "allowed_approvals_reviewers = [\"auto_review\"]\n")
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let requirements =
|
||||||
|
format!("{reviewer_requirements}[auto_review]\nrequired_on_models = [\"{model}\"]\n");
|
||||||
|
let mut extensions = ExtensionRegistryBuilder::new();
|
||||||
|
extensions.approval_review_contributor(Arc::new(ApprovedReviewContributor));
|
||||||
|
let mut builder = test_codex()
|
||||||
|
.with_model(model)
|
||||||
|
.with_extensions(Arc::new(extensions.build()))
|
||||||
|
.with_pre_build_hook(move |home| {
|
||||||
|
std::fs::write(
|
||||||
|
home.join("config.toml"),
|
||||||
|
format!("[features]\nguardianv2 = {guardian_v2_config}\n"),
|
||||||
|
)
|
||||||
|
.expect("Guardian v2 configuration should be written");
|
||||||
|
})
|
||||||
|
.with_cloud_config_bundle(
|
||||||
|
CloudConfigBundleFixture::loader_with_enterprise_requirement(requirements),
|
||||||
|
)
|
||||||
|
.with_config(|config| {
|
||||||
|
config.approvals_reviewer = ApprovalsReviewer::AutoReview;
|
||||||
|
config.permissions.approval_policy = Constrained::allow_any(AskForApproval::OnRequest);
|
||||||
|
config
|
||||||
|
.permissions
|
||||||
|
.set_permission_profile(PermissionProfile::read_only())
|
||||||
|
.expect("set read-only permission profile");
|
||||||
|
config
|
||||||
|
.features
|
||||||
|
.enable(Feature::RequestPermissionsTool)
|
||||||
|
.expect("test config should allow feature update");
|
||||||
|
});
|
||||||
|
let test = builder.build_with_auto_env(&server).await?;
|
||||||
|
assert!(!test.config.features.enabled(Feature::GuardianV2));
|
||||||
|
|
||||||
|
test.codex
|
||||||
|
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
|
||||||
|
text: reason.into(),
|
||||||
|
text_elements: Vec::new(),
|
||||||
|
}]))
|
||||||
|
.await?;
|
||||||
|
let event = wait_for_event(&test.codex, |event| {
|
||||||
|
matches!(
|
||||||
|
event,
|
||||||
|
EventMsg::RequestPermissions(_) | EventMsg::TurnComplete(_)
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
assert!(
|
||||||
|
matches!(event, EventMsg::TurnComplete(_)),
|
||||||
|
"required-model review should not prompt the user: {event:?}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let output = responses
|
||||||
|
.function_call_output_text(call_id)
|
||||||
|
.context("expected request_permissions output")?;
|
||||||
|
let actual: RequestPermissionsResponse = serde_json::from_str(&output)?;
|
||||||
|
assert_eq!(
|
||||||
|
actual,
|
||||||
|
RequestPermissionsResponse {
|
||||||
|
permissions: RequestPermissionProfile::default(),
|
||||||
|
scope: PermissionGrantScope::Turn,
|
||||||
|
strict_auto_review: false,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let requests = responses.requests();
|
||||||
|
let guardian_request = requests
|
||||||
|
.iter()
|
||||||
|
.find(|request| {
|
||||||
|
request.body_json()["client_metadata"]["x-openai-subagent"].as_str() == Some("guardian")
|
||||||
|
})
|
||||||
|
.context("expected full Guardian review for the required model")?;
|
||||||
|
assert!(guardian_request.body_contains_text(reason));
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
async fn remote_model_override_uses_catalog_model_for_strict_auto_review() -> Result<()> {
|
async fn remote_model_override_uses_catalog_model_for_strict_auto_review() -> Result<()> {
|
||||||
skip_if_no_network!(Ok(()));
|
skip_if_no_network!(Ok(()));
|
||||||
|
|||||||
Reference in new issue
Block a user