mirror of
https://github.com/openai/codex.git
synced 2026-11-16 06:54:36 +00:00
Interrupt cyber model turns after one Guardian denial (#37190)
## What changed - Add a Guardian circuit-breaker policy for models whose catalog specialty is `cyber`. - Interrupt those models after the first Guardian denial while retaining the existing denial thresholds for other models. - Centralize the `cyber` model-specialty identifier for use across core, protocol, app-server, and TUI code. ## Testing - Cover the cyber-specific threshold in the circuit-breaker unit tests. - Verify end to end that a denied command is not executed and the turn is interrupted after one denial. GitOrigin-RevId: fad24c9e7aa280432580db1f0e39d13535779710
This commit is contained in:
1 parent
92b83e226d
commit
f141dc77f0
9 files changed
+200
-23
No files matched your search
@@ -47,7 +47,9 @@ pub(crate) use review_session::prompt_cache_key_override_for_review_session;
|
||||
|
||||
pub(crate) const GUARDIAN_REVIEW_TIMEOUT: Duration = Duration::from_secs(90);
|
||||
pub(crate) const GUARDIAN_REVIEWER_NAME: &str = "guardian";
|
||||
pub(crate) const MAX_CONSECUTIVE_CYBER_GUARDIAN_DENIALS_PER_TURN: u32 = 1;
|
||||
pub(crate) const MAX_CONSECUTIVE_GUARDIAN_DENIALS_PER_TURN: u32 = 3;
|
||||
pub(crate) const MAX_RECENT_CYBER_AUTO_REVIEW_DENIALS_PER_TURN: u32 = 1;
|
||||
pub(crate) const MAX_RECENT_AUTO_REVIEW_DENIALS_PER_TURN: u32 = 10;
|
||||
pub(crate) const AUTO_REVIEW_DENIAL_WINDOW_SIZE: usize = 50;
|
||||
pub(crate) const AUTO_REVIEW_DENIED_ACTION_APPROVAL_DEVELOPER_PREFIX: &str =
|
||||
@@ -81,6 +83,12 @@ struct GuardianRejectionCircuitBreakerTurn {
|
||||
interrupt_triggered: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum GuardianRejectionCircuitBreakerPolicy {
|
||||
Standard,
|
||||
CyberModel,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum GuardianRejectionCircuitBreakerAction {
|
||||
Continue,
|
||||
@@ -95,14 +103,28 @@ impl GuardianRejectionCircuitBreaker {
|
||||
self.turns.remove(turn_id);
|
||||
}
|
||||
|
||||
pub(crate) fn record_denial(&mut self, turn_id: &str) -> GuardianRejectionCircuitBreakerAction {
|
||||
pub(crate) fn record_denial(
|
||||
&mut self,
|
||||
turn_id: &str,
|
||||
policy: GuardianRejectionCircuitBreakerPolicy,
|
||||
) -> GuardianRejectionCircuitBreakerAction {
|
||||
let turn = self.turns.entry(turn_id.to_string()).or_default();
|
||||
turn.consecutive_denials = turn.consecutive_denials.saturating_add(1);
|
||||
Self::record_recent_review(turn, /*denied*/ true);
|
||||
let recent_denials = turn.recent_denials.iter().filter(|denied| **denied).count() as u32;
|
||||
let (max_consecutive_denials, max_recent_denials) = match policy {
|
||||
GuardianRejectionCircuitBreakerPolicy::Standard => (
|
||||
MAX_CONSECUTIVE_GUARDIAN_DENIALS_PER_TURN,
|
||||
MAX_RECENT_AUTO_REVIEW_DENIALS_PER_TURN,
|
||||
),
|
||||
GuardianRejectionCircuitBreakerPolicy::CyberModel => (
|
||||
MAX_CONSECUTIVE_CYBER_GUARDIAN_DENIALS_PER_TURN,
|
||||
MAX_RECENT_CYBER_AUTO_REVIEW_DENIALS_PER_TURN,
|
||||
),
|
||||
};
|
||||
if !turn.interrupt_triggered
|
||||
&& (turn.consecutive_denials >= MAX_CONSECUTIVE_GUARDIAN_DENIALS_PER_TURN
|
||||
|| recent_denials >= MAX_RECENT_AUTO_REVIEW_DENIALS_PER_TURN)
|
||||
&& (turn.consecutive_denials >= max_consecutive_denials
|
||||
|| recent_denials >= max_recent_denials)
|
||||
{
|
||||
turn.interrupt_triggered = true;
|
||||
GuardianRejectionCircuitBreakerAction::InterruptTurn {
|
||||
|
||||
@@ -7,6 +7,7 @@ use codex_analytics::GuardianReviewTrackContext;
|
||||
use codex_analytics::GuardianReviewedAction;
|
||||
use codex_core_plugins::PluginCommandAttribution;
|
||||
use codex_protocol::config_types::ApprovalsReviewer;
|
||||
use codex_protocol::openai_models::MODEL_SPECIALTY_CYBER;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::CodexErrorInfo;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
@@ -38,6 +39,7 @@ use super::GuardianApprovalRequest;
|
||||
use super::GuardianAssessment;
|
||||
use super::GuardianAssessmentOutcome;
|
||||
use super::GuardianRejectionCircuitBreakerAction;
|
||||
use super::GuardianRejectionCircuitBreakerPolicy;
|
||||
use super::approval_request::guardian_assessment_action;
|
||||
use super::approval_request::guardian_request_target_item_id;
|
||||
use super::approval_request::guardian_request_turn_id;
|
||||
@@ -235,12 +237,17 @@ async fn record_guardian_non_denial(session: &Arc<Session>, turn_id: &str) {
|
||||
}
|
||||
|
||||
async fn record_guardian_denial(session: &Arc<Session>, turn: &Arc<TurnContext>, turn_id: &str) {
|
||||
let policy = if turn.model_info.model_specialty.as_deref() == Some(MODEL_SPECIALTY_CYBER) {
|
||||
GuardianRejectionCircuitBreakerPolicy::CyberModel
|
||||
} else {
|
||||
GuardianRejectionCircuitBreakerPolicy::Standard
|
||||
};
|
||||
let action = session
|
||||
.services
|
||||
.guardian_rejection_circuit_breaker
|
||||
.lock()
|
||||
.await
|
||||
.record_denial(turn_id);
|
||||
.record_denial(turn_id, policy);
|
||||
let GuardianRejectionCircuitBreakerAction::InterruptTurn {
|
||||
consecutive_denials,
|
||||
recent_denials,
|
||||
|
||||
@@ -139,22 +139,38 @@ impl codex_extension_api::ContextContributor for GuardianMemoryContextProbe {
|
||||
fn guardian_rejection_circuit_breaker_interrupts_after_three_consecutive_denials() {
|
||||
let mut circuit_breaker = GuardianRejectionCircuitBreaker::default();
|
||||
assert_eq!(
|
||||
circuit_breaker.record_denial("turn-1"),
|
||||
circuit_breaker.record_denial("turn-1", GuardianRejectionCircuitBreakerPolicy::Standard),
|
||||
GuardianRejectionCircuitBreakerAction::Continue
|
||||
);
|
||||
assert_eq!(
|
||||
circuit_breaker.record_denial("turn-1"),
|
||||
circuit_breaker.record_denial("turn-1", GuardianRejectionCircuitBreakerPolicy::Standard),
|
||||
GuardianRejectionCircuitBreakerAction::Continue
|
||||
);
|
||||
assert_eq!(
|
||||
circuit_breaker.record_denial("turn-1"),
|
||||
circuit_breaker.record_denial("turn-1", GuardianRejectionCircuitBreakerPolicy::Standard),
|
||||
GuardianRejectionCircuitBreakerAction::InterruptTurn {
|
||||
consecutive_denials: 3,
|
||||
recent_denials: 3,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
circuit_breaker.record_denial("turn-1"),
|
||||
circuit_breaker.record_denial("turn-1", GuardianRejectionCircuitBreakerPolicy::Standard),
|
||||
GuardianRejectionCircuitBreakerAction::Continue
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guardian_rejection_circuit_breaker_interrupts_cyber_models_after_one_denial() {
|
||||
let mut circuit_breaker = GuardianRejectionCircuitBreaker::default();
|
||||
assert_eq!(
|
||||
circuit_breaker.record_denial("turn-1", GuardianRejectionCircuitBreakerPolicy::CyberModel),
|
||||
GuardianRejectionCircuitBreakerAction::InterruptTurn {
|
||||
consecutive_denials: 1,
|
||||
recent_denials: 1,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
circuit_breaker.record_denial("turn-1", GuardianRejectionCircuitBreakerPolicy::CyberModel),
|
||||
GuardianRejectionCircuitBreakerAction::Continue
|
||||
);
|
||||
}
|
||||
@@ -163,20 +179,20 @@ fn guardian_rejection_circuit_breaker_interrupts_after_three_consecutive_denials
|
||||
fn guardian_rejection_circuit_breaker_resets_consecutive_denials_on_non_denial() {
|
||||
let mut circuit_breaker = GuardianRejectionCircuitBreaker::default();
|
||||
assert_eq!(
|
||||
circuit_breaker.record_denial("turn-1"),
|
||||
circuit_breaker.record_denial("turn-1", GuardianRejectionCircuitBreakerPolicy::Standard),
|
||||
GuardianRejectionCircuitBreakerAction::Continue
|
||||
);
|
||||
circuit_breaker.record_non_denial("turn-1");
|
||||
assert_eq!(
|
||||
circuit_breaker.record_denial("turn-1"),
|
||||
circuit_breaker.record_denial("turn-1", GuardianRejectionCircuitBreakerPolicy::Standard),
|
||||
GuardianRejectionCircuitBreakerAction::Continue
|
||||
);
|
||||
assert_eq!(
|
||||
circuit_breaker.record_denial("turn-1"),
|
||||
circuit_breaker.record_denial("turn-1", GuardianRejectionCircuitBreakerPolicy::Standard),
|
||||
GuardianRejectionCircuitBreakerAction::Continue
|
||||
);
|
||||
assert_eq!(
|
||||
circuit_breaker.record_denial("turn-1"),
|
||||
circuit_breaker.record_denial("turn-1", GuardianRejectionCircuitBreakerPolicy::Standard),
|
||||
GuardianRejectionCircuitBreakerAction::InterruptTurn {
|
||||
consecutive_denials: 3,
|
||||
recent_denials: 4,
|
||||
@@ -189,13 +205,14 @@ fn auto_review_rejection_circuit_breaker_interrupts_after_ten_recent_denials() {
|
||||
let mut circuit_breaker = GuardianRejectionCircuitBreaker::default();
|
||||
for _ in 0..9 {
|
||||
assert_eq!(
|
||||
circuit_breaker.record_denial("turn-1"),
|
||||
circuit_breaker
|
||||
.record_denial("turn-1", GuardianRejectionCircuitBreakerPolicy::Standard),
|
||||
GuardianRejectionCircuitBreakerAction::Continue
|
||||
);
|
||||
circuit_breaker.record_non_denial("turn-1");
|
||||
}
|
||||
assert_eq!(
|
||||
circuit_breaker.record_denial("turn-1"),
|
||||
circuit_breaker.record_denial("turn-1", GuardianRejectionCircuitBreakerPolicy::Standard),
|
||||
GuardianRejectionCircuitBreakerAction::InterruptTurn {
|
||||
consecutive_denials: 1,
|
||||
recent_denials: 10,
|
||||
@@ -208,7 +225,8 @@ fn auto_review_rejection_circuit_breaker_forgets_denials_outside_recent_review_w
|
||||
let mut circuit_breaker = GuardianRejectionCircuitBreaker::default();
|
||||
for _ in 0..9 {
|
||||
assert_eq!(
|
||||
circuit_breaker.record_denial("turn-1"),
|
||||
circuit_breaker
|
||||
.record_denial("turn-1", GuardianRejectionCircuitBreakerPolicy::Standard),
|
||||
GuardianRejectionCircuitBreakerAction::Continue
|
||||
);
|
||||
circuit_breaker.record_non_denial("turn-1");
|
||||
@@ -217,7 +235,7 @@ fn auto_review_rejection_circuit_breaker_forgets_denials_outside_recent_review_w
|
||||
circuit_breaker.record_non_denial("turn-1");
|
||||
}
|
||||
assert_eq!(
|
||||
circuit_breaker.record_denial("turn-1"),
|
||||
circuit_breaker.record_denial("turn-1", GuardianRejectionCircuitBreakerPolicy::Standard),
|
||||
GuardianRejectionCircuitBreakerAction::Continue
|
||||
);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user