Block goals after repeated execution host failures (#41454)

## What changed

- Track failed `exec` attempts whose handler ran for each active goal.
- Mark the goal as blocked after three qualifying failure turns.
- Reset the failure streak when any tool succeeds, and keep failures from carrying over to a replacement goal.

## Testing

- Cover failure classification, streak resets, goal replacement, and end-to-end goal blocking through the app server.

GitOrigin-RevId: 14dac7f4eb35507f12df3b2a82ab97bdafe7b897

codex-rs/ext/goal/src/accounting.rs

@@ -1,3 +1,5 @@
+use codex_extension_api::ToolCallOutcome;
+use codex_extension_api::ToolName;
 use codex_protocol::config_types::ModeKind;
 use codex_protocol::protocol::TokenUsage;
 use codex_state::ThreadGoalStatus;
@@ -24,6 +26,8 @@ struct GoalAccountingInner {
     turns: HashMap<String, GoalTurnAccounting>,
     wall_clock: GoalWallClockAccounting,
     budget_limit_reported_goal_id: Option<String>,
+    execution_failure_goal_id: Option<String>,
+    consecutive_execution_failure_turns: u8,
     last_accounted_descendant_token_usage: i64,
 }
 
@@ -33,6 +37,8 @@ struct GoalTurnAccounting {
     last_accounted_token_usage: TokenUsage,
     active_goal_id: Option<String>,
     account_tokens: bool,
+    failed_execution: bool,
+    successful_tool: bool,
 }
 
 #[derive(Debug)]
@@ -93,6 +99,58 @@ impl GoalAccountingState {
         self.inner().current_turn_id.clone()
     }
 
+    pub(crate) fn record_tool_outcome(
+        &self,
+        turn_id: &str,
+        tool_name: &ToolName,
+        outcome: ToolCallOutcome,
+    ) {
+        let mut inner = self.inner();
+        let Some(turn) = inner.turns.get_mut(turn_id) else {
+            return;
+        };
+        if turn.active_goal_id.is_none() {
+            return;
+        }
+
+        match outcome {
+            ToolCallOutcome::Completed { success: true } => {
+                turn.successful_tool = true;
+                inner.execution_failure_goal_id = None;
+                inner.consecutive_execution_failure_turns = 0;
+            }
+            ToolCallOutcome::Failed {
+                handler_executed: true,
+            } if tool_name.is_default_namespace() && tool_name.name == "exec" => {
+                turn.failed_execution = true;
+            }
+            ToolCallOutcome::Completed { success: false }
+            | ToolCallOutcome::Failed { .. }
+            | ToolCallOutcome::Blocked
+            | ToolCallOutcome::Aborted => {}
+        }
+    }
+
+    pub(crate) fn execution_failure_goal(&self, turn_id: &str) -> Option<String> {
+        let mut inner = self.inner();
+        let turn = inner.turns.get(turn_id)?;
+        let goal_id = turn.active_goal_id.clone()?;
+        if turn.successful_tool {
+            return None;
+        }
+        if !turn.failed_execution {
+            return None;
+        }
+
+        if inner.execution_failure_goal_id.as_deref() != Some(goal_id.as_str()) {
+            inner.execution_failure_goal_id = Some(goal_id.clone());
+            inner.consecutive_execution_failure_turns = 0;
+        }
+        inner.consecutive_execution_failure_turns =
+            inner.consecutive_execution_failure_turns.saturating_add(1);
+        (inner.consecutive_execution_failure_turns >= 3).then_some(goal_id)
+    }
+
     /// Acquires the per-thread progress-accounting permit.
     ///
     /// Hold the returned permit from before taking a progress snapshot until after the persistent
@@ -104,15 +162,16 @@ impl GoalAccountingState {
         self.progress_accounting_lock.acquire().await
     }
 
-    pub(crate) fn turn_is_current_active_goal(&self, turn_id: &str) -> bool {
+    pub(crate) fn current_active_goal_id_for_turn(&self, turn_id: &str) -> Option<String> {
         let inner = self.inner();
         if inner.current_turn_id.as_deref() != Some(turn_id) {
-            return false;
+            return None;
         }
-        let Some(turn) = inner.turns.get(turn_id) else {
-            return false;
-        };
-        turn.account_tokens && turn.active_goal_id.is_some()
+        let turn = inner.turns.get(turn_id)?;
+        if !turn.account_tokens {
+            return None;
+        }
+        turn.active_goal_id.clone()
     }
 
     pub(crate) fn record_token_usage(
@@ -176,6 +235,10 @@ impl GoalAccountingState {
         }
         let goal_changed = inner.wall_clock.active_goal_id.as_deref() != Some(goal_id.as_str());
         let turn = inner.turns.get_mut(turn_id.as_str())?;
+        if turn.active_goal_id.as_deref() != Some(goal_id.as_str()) {
+            turn.failed_execution = false;
+            turn.successful_tool = false;
+        }
         turn.active_goal_id = Some(goal_id.clone());
         if goal_changed {
             turn.reset_baseline_to_current();
@@ -207,6 +270,8 @@ impl GoalAccountingState {
         }
         inner.wall_clock.clear_active_goal();
         inner.budget_limit_reported_goal_id = None;
+        inner.execution_failure_goal_id = None;
+        inner.consecutive_execution_failure_turns = 0;
         Some(turn_id)
     }
 
@@ -219,6 +284,8 @@ impl GoalAccountingState {
         }
         inner.wall_clock.clear_active_goal();
         inner.budget_limit_reported_goal_id = None;
+        inner.execution_failure_goal_id = None;
+        inner.consecutive_execution_failure_turns = 0;
     }
 
     pub(crate) fn progress_snapshot(&self, turn_id: &str) -> Option<GoalProgressSnapshot> {
@@ -385,6 +452,8 @@ impl Default for GoalAccountingInner {
             turns: HashMap::new(),
             wall_clock: GoalWallClockAccounting::new(),
             budget_limit_reported_goal_id: None,
+            execution_failure_goal_id: None,
+            consecutive_execution_failure_turns: 0,
             last_accounted_descendant_token_usage: 0,
         }
     }
@@ -408,6 +477,8 @@ impl GoalTurnAccounting {
             current_token_usage,
             active_goal_id: None,
             account_tokens,
+            failed_execution: false,
+            successful_tool: false,
         }
     }
 

codex-rs/ext/goal/src/extension.rs

@@ -279,6 +279,20 @@ where
             }
 
             let turn_id = input.turn_store.level_id();
+            if let Some(expected_goal_id) =
+                runtime.accounting_state().execution_failure_goal(turn_id)
+                && let Err(err) = runtime
+                    .stop_active_goal_for_turn(
+                        turn_id,
+                        ActiveGoalStopReason::ExecutionUnavailable { expected_goal_id },
+                    )
+                    .await
+            {
+                tracing::warn!(
+                    "failed to stop active goal after repeated execution failures for {turn_id}: {err}"
+                );
+                return;
+            }
             if let Err(err) = runtime
                 .account_active_goal_progress(
                     turn_id,
@@ -390,10 +404,18 @@ where
             let Some(runtime) = goal_runtime_handle(input.thread_store) else {
                 return;
             };
-            let should_count_for_goal_progress = runtime.is_enabled()
-                && tool_attempt_counts_for_goal_progress(input.outcome)
-                && !(input.tool_name.is_default_namespace()
-                    && input.tool_name.name == UPDATE_GOAL_TOOL_NAME);
+            if !runtime.is_enabled() {
+                return;
+            }
+            runtime.accounting_state().record_tool_outcome(
+                input.turn_id,
+                input.tool_name,
+                input.outcome,
+            );
+            let should_count_for_goal_progress =
+                tool_attempt_counts_for_goal_progress(input.outcome)
+                    && !(input.tool_name.is_default_namespace()
+                        && input.tool_name.name == UPDATE_GOAL_TOOL_NAME);
             if !should_count_for_goal_progress {
                 return;
             }

codex-rs/ext/goal/src/runtime.rs

@@ -39,6 +39,7 @@ pub(crate) struct GoalRuntimeConfig {
 pub(crate) enum ActiveGoalStopReason {
     TurnError,
     UsageLimit,
+    ExecutionUnavailable { expected_goal_id: String },
 }
 
 struct GoalRuntimeInner {
@@ -264,21 +265,33 @@ impl GoalRuntimeHandle {
         // Hold this through accounting and the status update so external goal
         // mutations and idle continuation cannot interleave between them.
         let _goal_state_permit = self.goal_state_permit().await?;
-        if !self
+        let Some(accounting_goal_id) = self
             .inner
             .accounting_state
-            .turn_is_current_active_goal(turn_id)
+            .current_active_goal_id_for_turn(turn_id)
+        else {
+            return Ok(());
+        };
+        if let ActiveGoalStopReason::ExecutionUnavailable { expected_goal_id } = &reason
+            && accounting_goal_id != *expected_goal_id
         {
             return Ok(());
         }
 
-        let (event_name, status) = match reason {
+        let (event_name, status, expected_goal_id) = match reason {
             ActiveGoalStopReason::TurnError => {
-                ("turn-error", codex_state::ThreadGoalStatus::Blocked)
-            }
-            ActiveGoalStopReason::UsageLimit => {
-                ("usage-limit", codex_state::ThreadGoalStatus::UsageLimited)
+                ("turn-error", codex_state::ThreadGoalStatus::Blocked, None)
             }
+            ActiveGoalStopReason::UsageLimit => (
+                "usage-limit",
+                codex_state::ThreadGoalStatus::UsageLimited,
+                None,
+            ),
+            ActiveGoalStopReason::ExecutionUnavailable { expected_goal_id } => (
+                "execution-unavailable",
+                codex_state::ThreadGoalStatus::Blocked,
+                Some(expected_goal_id),
+            ),
         };
         self.account_active_goal_progress(
             turn_id,
@@ -299,6 +312,12 @@ impl GoalRuntimeHandle {
             self.inner.accounting_state.clear_active_goal();
             return Ok(());
         };
+        if expected_goal_id
+            .as_ref()
+            .is_some_and(|expected_goal_id| active_goal.goal_id != *expected_goal_id)
+        {
+            return Ok(());
+        }
         let can_stop = active_goal.status == codex_state::ThreadGoalStatus::Active
             || (active_goal.status == codex_state::ThreadGoalStatus::BudgetLimited
                 && status == codex_state::ThreadGoalStatus::UsageLimited);
@@ -444,10 +463,13 @@ impl GoalRuntimeHandle {
             .is_some_and(|turn_id| {
                 self.inner
                     .accounting_state
-                    .turn_is_current_active_goal(turn_id.as_str())
+                    .current_active_goal_id_for_turn(turn_id.as_str())
+                    .is_some()
             });
         if !current_turn_is_goal_active {
-            self.inner.accounting_state.clear_active_goal();
+            self.inner
+                .accounting_state
+                .reset_idle_progress_baseline_and_clear_active_goal();
         }
         Ok(())
     }