Snapshot resolved settings for each model step (#40651)

## Why

Thread settings can change while a turn is running, and delegated review or
compaction steps can select a different model. Each request needs a consistent
set of settings resolved against the model that will execute it.

## What changed

- Capture immutable resolved settings for each model step, including model
  metadata, reasoning options, service tier, approvals, and personality.
- Keep in-flight steps on their captured settings while applying thread updates
  to subsequent work.
- Resolve inherited reasoning summaries and filter service tiers against the
  selected model and feature configuration for review and compaction steps.

## Testing

Added coverage for settings updates during paused turns, model changes,
previous-model compaction, review-model defaults, and service-tier filtering.

GitOrigin-RevId: fca4b3b305379f1bf26a5cac75ee0dd1747ac1f6
This commit is contained in:
joeflorencio-openai authored and copyberry committed 2026-08-25 17:41:22 +00:00
1 parent ba6cf9c692
commit 68301fa45f
68 files changed
+1393 -516

No files matched your search

+94
View File
@@ -14,10 +14,13 @@ use codex_models_manager::bundled_models_response;
use codex_protocol::config_types::AutoCompactTokenLimitScope;
use codex_protocol::config_types::CollaborationMode;
use codex_protocol::config_types::ModeKind;
use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::config_types::ServiceTier;
use codex_protocol::config_types::Settings;
use codex_protocol::items::TurnItem;
use codex_protocol::models::PermissionProfile;
use codex_protocol::openai_models::ModelInfo;
use codex_protocol::openai_models::ModelServiceTier;
use codex_protocol::openai_models::ModelsResponse;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::EventMsg;
@@ -2280,6 +2283,97 @@ async fn pre_sampling_compact_runs_when_comp_hash_changes() {
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn previous_model_compaction_resolves_selected_settings() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let request_log = mount_sse_sequence(
&server,
vec![
sse(vec![
ev_assistant_message("m1", "before switch"),
ev_completed("r1"),
]),
sse(vec![
ev_assistant_message("m2", "COMPACTION_SUMMARY"),
ev_completed("r2"),
]),
responses::sse_completed("r3"),
],
)
.await;
let model_provider = non_openai_model_provider(&server);
let test = test_codex()
.with_model_info_override("gpt-5.4", |model| {
model.comp_hash = Some("hash-a".to_string());
model.default_reasoning_summary = ReasoningSummary::Detailed;
model.service_tiers = vec![ModelServiceTier {
id: ServiceTier::Fast.request_value().to_string(),
name: "Fast".to_string(),
description: "Priority processing".to_string(),
}];
})
.with_model_info_override("gpt-5.2", |model| {
model.comp_hash = Some("hash-b".to_string());
model.default_reasoning_summary = ReasoningSummary::Auto;
model.service_tiers.clear();
})
.with_model("gpt-5.4")
.with_config(move |config| {
config.model_provider = model_provider;
config.model_reasoning_summary = None;
config.service_tier = Some(ServiceTier::Fast.request_value().to_string());
config
.features
.enable(Feature::FastMode)
.expect("enable FastMode");
set_test_compact_prompt(config);
})
.build_with_auto_env(&server)
.await?;
test.submit_text_turn("before switch").await?;
test.codex
.start_or_steer_turn(
TurnInputRequest::user_input(vec![UserInput::Text {
text: "after switch".to_string(),
text_elements: Vec::new(),
}])
.with_thread_settings(ThreadSettingsOverrides {
model: Some("gpt-5.2".to_string()),
..Default::default()
}),
)
.await?;
wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
.await;
// The retained priority selection is filtered out for B, restored for
// compaction on A, and still omitted from the following B request.
let actual = request_log
.requests()
.iter()
.map(|request| {
let body = request.body_json();
json!([
body["model"],
body["reasoning"]["summary"],
body["service_tier"]
])
})
.collect::<Vec<_>>();
assert_eq!(
actual,
vec![
json!(["gpt-5.4", "detailed", "priority"]),
json!(["gpt-5.4", "detailed", "priority"]),
json!(["gpt-5.2", "auto", null]),
]
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn pre_sampling_compact_falls_back_from_retired_previous_model_after_rename() {
skip_if_no_network!();
+1
View File
@@ -153,6 +153,7 @@ mod skills;
mod skills_extension;
mod spawn_agent_description;
mod sqlite_state;
mod step_settings_snapshots;
mod stream_error_allows_next_turn;
mod stream_no_completed;
mod subagent_notifications;
+167 -5
View File
@@ -10,7 +10,12 @@ use codex_history::RolloutItem;
use codex_history::RolloutLine;
use codex_login::CodexAuth;
use codex_protocol::config_types::ApprovalsReviewer;
use codex_protocol::config_types::CollaborationMode;
use codex_protocol::config_types::ModeKind;
use codex_protocol::config_types::Personality;
use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::config_types::ServiceTier;
use codex_protocol::config_types::Settings;
use codex_protocol::items::TurnItem;
use codex_protocol::models::ContentItem;
use codex_protocol::models::PermissionProfile;
@@ -627,6 +632,10 @@ async fn review_uses_updated_turn_permissions_and_approval_policy() {
config.review_model = Some("gpt-5.4".to_string());
config.model_context_window = Some(128_000);
config.service_tier = Some(ServiceTier::Fast.request_value().to_string());
config
.features
.enable(Feature::FastMode)
.expect("enable FastMode");
config
.features
.enable(Feature::TokenBudget)
@@ -671,13 +680,22 @@ async fn review_uses_updated_turn_permissions_and_approval_policy() {
approval_policy: Some(AskForApproval::Never),
approvals_reviewer: Some(ApprovalsReviewer::User),
permission_profile: Some(PermissionProfile::Disabled),
effort: Some(Some(ReasoningEffort::XHigh)),
personality: Some(Personality::Friendly),
collaboration_mode: Some(CollaborationMode {
mode: ModeKind::Plan,
settings: Settings {
model: "gpt-5.2".to_string(),
reasoning_effort: Some(ReasoningEffort::XHigh),
developer_instructions: Some("Parent planning instructions".to_string()),
},
}),
..Default::default()
},
)
.await
.expect("updated thread permissions should be accepted");
let stored_settings = codex.thread_settings_snapshot().await;
codex
.submit(Op::Review {
review_request: ReviewRequest {
@@ -691,6 +709,7 @@ async fn review_uses_updated_turn_permissions_and_approval_policy() {
.expect("review should start");
wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
assert_eq!(codex.thread_settings_snapshot().await, stored_settings);
let request = request_log.single_request();
assert_eq!(request.body_json()["reasoning"]["effort"], "medium");
assert_eq!(
@@ -751,23 +770,166 @@ async fn review_uses_updated_turn_permissions_and_approval_policy() {
})
.expect("review rollout should contain session metadata");
assert_eq!(review_session_cwd, updated_cwd.as_path());
let review_approvals_reviewer = review_rollout
let review_context = review_rollout
.lines()
.filter_map(|line| {
let rollout_line: RolloutLine =
serde_json::from_str(line).expect("review rollout line should be valid");
match rollout_line.item {
RolloutItem::TurnContext(turn_context) => turn_context.approvals_reviewer,
RolloutItem::TurnContext(turn_context) => Some(turn_context),
_ => None,
}
})
.next_back();
assert_eq!(review_approvals_reviewer, Some(ApprovalsReviewer::User));
.next_back()
.expect("review rollout should contain turn context");
assert_eq!(
review_context.approvals_reviewer,
Some(ApprovalsReviewer::User)
);
assert_eq!(review_context.personality, Some(Personality::Friendly));
// The review delegate still starts in its own default mode, not the parent's Plan mode.
assert_eq!(
review_context.collaboration_mode,
Some(CollaborationMode {
mode: ModeKind::Default,
settings: Settings {
model: "gpt-5.4".to_string(),
reasoning_effort: Some(ReasoningEffort::Medium),
developer_instructions: None,
},
})
);
let _codex_home_guard = codex_home;
server.verify().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn review_omits_retained_tier_when_fast_mode_disabled() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let (server, request_log) =
start_responses_server_with_sse(completed_sse(), /*expected_requests*/ 1).await;
let test = test_codex()
.with_model_info_override("gpt-5.4", |model| {
model.service_tiers = vec![ModelServiceTier {
id: ServiceTier::Flex.request_value().to_string(),
name: "Flex".to_string(),
description: "Flexible processing".to_string(),
}];
})
.with_config(|config| {
config
.features
.disable(Feature::FastMode)
.expect("disable FastMode");
})
.build_with_auto_env(&server)
.await?;
core_test_support::submit_thread_settings(
&test.codex,
ThreadSettingsOverrides {
service_tier: Some(Some(ServiceTier::Flex.request_value().to_string())),
..Default::default()
},
)
.await?;
test.codex
.submit(Op::Review {
review_request: ReviewRequest {
target: ReviewTarget::Custom {
instructions: "review the changes".to_string(),
},
user_facing_hint: None,
},
})
.await?;
wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
.await;
assert_eq!(
test.codex
.thread_settings_snapshot()
.await
.service_tier
.as_deref(),
Some("flex")
);
assert_eq!(
request_log.single_request().body_json().get("service_tier"),
None
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn review_resolves_inherited_summary_preferences() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let (server, request_log) =
start_responses_server_with_sse(completed_sse(), /*expected_requests*/ 2).await;
let test = test_codex()
.with_model_info_override("gpt-5.2", |model| {
model.default_reasoning_summary = ReasoningSummary::Auto;
})
.with_model_info_override("gpt-5.4", |model| {
model.default_reasoning_summary = ReasoningSummary::Detailed;
})
.with_model("gpt-5.2")
.with_config(|config| {
config.review_model = Some("gpt-5.4".to_string());
config.model_reasoning_summary = None;
})
.build_with_auto_env(&server)
.await?;
// First follow the review model's default, then use a preference updated on the thread.
for summary in [None, Some(ReasoningSummary::Concise)] {
if let Some(summary) = summary {
core_test_support::submit_thread_settings(
&test.codex,
ThreadSettingsOverrides {
summary: Some(summary),
..Default::default()
},
)
.await?;
}
let stored_settings = test.codex.thread_settings_snapshot().await;
test.codex
.submit(Op::Review {
review_request: ReviewRequest {
target: ReviewTarget::Custom {
instructions: "review the changes".to_string(),
},
user_facing_hint: None,
},
})
.await?;
wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
.await;
assert_eq!(test.codex.thread_settings_snapshot().await, stored_settings);
}
let actual = request_log
.requests()
.iter()
.map(|request| {
let body = request.body_json();
serde_json::json!([body["model"], body["reasoning"]["summary"]])
})
.collect::<Vec<_>>();
assert_eq!(
actual,
vec![
serde_json::json!(["gpt-5.4", "detailed"]),
serde_json::json!(["gpt-5.4", "concise"]),
]
);
Ok(())
}
/// Ensure that when a custom `review_model` is set in the config, the review
/// request uses that model (and not the main chat model).
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -0,0 +1,171 @@
use anyhow::Result;
use codex_core::TurnInputRequest;
use codex_features::Feature;
use codex_models_manager::bundled_models_response;
use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::config_types::ServiceTier;
use codex_protocol::openai_models::ModelsResponse;
use codex_protocol::openai_models::ReasoningEffort;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::Op;
use codex_protocol::protocol::ThreadSettingsOverrides;
use codex_protocol::request_user_input::RequestUserInputAnswer;
use codex_protocol::request_user_input::RequestUserInputResponse;
use codex_protocol::user_input::UserInput;
use core_test_support::responses;
use core_test_support::skip_if_no_network;
use core_test_support::submit_thread_settings;
use core_test_support::test_codex::test_codex;
use core_test_support::wait_for_event;
use core_test_support::wait_for_event_match;
use pretty_assertions::assert_eq;
use serde_json::json;
use std::collections::HashMap;
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn ordinary_settings_updates_keep_the_running_turn_snapshot() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = responses::start_mock_server().await;
let response_mock = responses::mount_sse_sequence(
&server,
vec![
responses::sse(vec![
responses::ev_response_created("response-1"),
responses::ev_function_call(
"pause",
"request_user_input",
&json!({ "questions": [{
"id": "continue",
"header": "Continue",
"question": "Continue after the settings update?",
"options": [
{ "label": "Yes", "description": "Continue the turn." },
{ "label": "No", "description": "Stop the turn." }
]
}] })
.to_string(),
),
responses::ev_completed("response-1"),
]),
responses::sse_completed("response-2"),
responses::sse_completed("response-3"),
],
)
.await;
let test = test_codex()
.with_model("snapshot-model-a")
.with_config(|config| {
// This regression exercises ordinary turn construction without
// enabling the diagnostic active-settings path.
config
.features
.enable(Feature::DefaultModeRequestUserInput)
.unwrap();
config.features.enable(Feature::FastMode).unwrap();
let template = bundled_models_response()
.unwrap()
.models
.into_iter()
.find(|model| model.slug == "gpt-5.4")
.unwrap();
config.model_catalog = Some(ModelsResponse {
models: ["snapshot-model-a", "snapshot-model-b"]
.into_iter()
.map(|slug| {
let mut model = template.clone();
model.slug = slug.to_string();
model.default_reasoning_summary = ReasoningSummary::Concise;
model
})
.collect(),
});
config.model_reasoning_effort = Some(ReasoningEffort::Low);
config.model_reasoning_summary = None;
config.service_tier = None;
})
.build_with_auto_env(&server)
.await?;
test.codex
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
text: "pause before continuing".into(),
text_elements: Vec::new(),
}]))
.await?;
let paused = wait_for_event_match(&test.codex, |event| match event {
EventMsg::RequestUserInput(request) => Some(request.clone()),
_ => None,
})
.await;
let mut expected = test.codex.thread_settings_snapshot().await;
submit_thread_settings(
&test.codex,
ThreadSettingsOverrides {
model: Some("snapshot-model-b".to_string()),
effort: Some(Some(ReasoningEffort::High)),
summary: Some(ReasoningSummary::Detailed),
service_tier: Some(Some(ServiceTier::Fast.request_value().to_string())),
..Default::default()
},
)
.await?;
expected.model = "snapshot-model-b".to_string();
expected.reasoning_effort = Some(ReasoningEffort::High);
expected.reasoning_summary = Some(ReasoningSummary::Detailed);
expected.service_tier = Some(ServiceTier::Fast.request_value().to_string());
expected.collaboration_mode = expected.collaboration_mode.with_updates(
Some(expected.model.clone()),
Some(expected.reasoning_effort.clone()),
/*developer_instructions*/ None,
);
assert_eq!(test.codex.thread_settings_snapshot().await, expected);
test.codex
.submit(Op::UserInputAnswer {
id: paused.turn_id.clone(),
response: RequestUserInputResponse {
answers: HashMap::from([(
"continue".to_string(),
RequestUserInputAnswer {
answers: vec!["Yes".to_string()],
},
)]),
},
})
.await?;
wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
.await;
test.submit_text_turn("start the next turn").await?;
let requests = response_mock.requests();
let actual = requests
.iter()
.map(|request| {
let body = request.body_json();
json!({
"model": body["model"],
"reasoning": body["reasoning"],
"service_tier": body.get("service_tier"),
})
})
.collect::<Vec<_>>();
let original = json!({
"model": "snapshot-model-a",
"reasoning": { "effort": "low", "summary": "concise" },
"service_tier": null,
});
assert_eq!(
actual,
vec![
original.clone(),
original,
json!({
"model": "snapshot-model-b",
"reasoning": { "effort": "high", "summary": "detailed" },
"service_tier": "priority",
})
]
);
Ok(())
}