mirror of
https://github.com/openai/codex.git
synced 2026-11-06 00:30:29 +00:00
Snapshot resolved settings for each model step (#40651)
## Why Thread settings can change while a turn is running, and delegated review or compaction steps can select a different model. Each request needs a consistent set of settings resolved against the model that will execute it. ## What changed - Capture immutable resolved settings for each model step, including model metadata, reasoning options, service tier, approvals, and personality. - Keep in-flight steps on their captured settings while applying thread updates to subsequent work. - Resolve inherited reasoning summaries and filter service tiers against the selected model and feature configuration for review and compaction steps. ## Testing Added coverage for settings updates during paused turns, model changes, previous-model compaction, review-model defaults, and service-tier filtering. GitOrigin-RevId: fca4b3b305379f1bf26a5cac75ee0dd1747ac1f6
This commit is contained in:
1 parent
ba6cf9c692
commit
68301fa45f
68 files changed
+1393
-516
No files matched your search
@@ -14,10 +14,13 @@ use codex_models_manager::bundled_models_response;
|
||||
use codex_protocol::config_types::AutoCompactTokenLimitScope;
|
||||
use codex_protocol::config_types::CollaborationMode;
|
||||
use codex_protocol::config_types::ModeKind;
|
||||
use codex_protocol::config_types::ReasoningSummary;
|
||||
use codex_protocol::config_types::ServiceTier;
|
||||
use codex_protocol::config_types::Settings;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::models::PermissionProfile;
|
||||
use codex_protocol::openai_models::ModelInfo;
|
||||
use codex_protocol::openai_models::ModelServiceTier;
|
||||
use codex_protocol::openai_models::ModelsResponse;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
@@ -2280,6 +2283,97 @@ async fn pre_sampling_compact_runs_when_comp_hash_changes() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn previous_model_compaction_resolves_selected_settings() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let server = start_mock_server().await;
|
||||
let request_log = mount_sse_sequence(
|
||||
&server,
|
||||
vec![
|
||||
sse(vec![
|
||||
ev_assistant_message("m1", "before switch"),
|
||||
ev_completed("r1"),
|
||||
]),
|
||||
sse(vec![
|
||||
ev_assistant_message("m2", "COMPACTION_SUMMARY"),
|
||||
ev_completed("r2"),
|
||||
]),
|
||||
responses::sse_completed("r3"),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
let model_provider = non_openai_model_provider(&server);
|
||||
let test = test_codex()
|
||||
.with_model_info_override("gpt-5.4", |model| {
|
||||
model.comp_hash = Some("hash-a".to_string());
|
||||
model.default_reasoning_summary = ReasoningSummary::Detailed;
|
||||
model.service_tiers = vec![ModelServiceTier {
|
||||
id: ServiceTier::Fast.request_value().to_string(),
|
||||
name: "Fast".to_string(),
|
||||
description: "Priority processing".to_string(),
|
||||
}];
|
||||
})
|
||||
.with_model_info_override("gpt-5.2", |model| {
|
||||
model.comp_hash = Some("hash-b".to_string());
|
||||
model.default_reasoning_summary = ReasoningSummary::Auto;
|
||||
model.service_tiers.clear();
|
||||
})
|
||||
.with_model("gpt-5.4")
|
||||
.with_config(move |config| {
|
||||
config.model_provider = model_provider;
|
||||
config.model_reasoning_summary = None;
|
||||
config.service_tier = Some(ServiceTier::Fast.request_value().to_string());
|
||||
config
|
||||
.features
|
||||
.enable(Feature::FastMode)
|
||||
.expect("enable FastMode");
|
||||
set_test_compact_prompt(config);
|
||||
})
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
test.submit_text_turn("before switch").await?;
|
||||
test.codex
|
||||
.start_or_steer_turn(
|
||||
TurnInputRequest::user_input(vec![UserInput::Text {
|
||||
text: "after switch".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}])
|
||||
.with_thread_settings(ThreadSettingsOverrides {
|
||||
model: Some("gpt-5.2".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
|
||||
// The retained priority selection is filtered out for B, restored for
|
||||
// compaction on A, and still omitted from the following B request.
|
||||
let actual = request_log
|
||||
.requests()
|
||||
.iter()
|
||||
.map(|request| {
|
||||
let body = request.body_json();
|
||||
json!([
|
||||
body["model"],
|
||||
body["reasoning"]["summary"],
|
||||
body["service_tier"]
|
||||
])
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
actual,
|
||||
vec![
|
||||
json!(["gpt-5.4", "detailed", "priority"]),
|
||||
json!(["gpt-5.4", "detailed", "priority"]),
|
||||
json!(["gpt-5.2", "auto", null]),
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn pre_sampling_compact_falls_back_from_retired_previous_model_after_rename() {
|
||||
skip_if_no_network!();
|
||||
|
||||
@@ -153,6 +153,7 @@ mod skills;
|
||||
mod skills_extension;
|
||||
mod spawn_agent_description;
|
||||
mod sqlite_state;
|
||||
mod step_settings_snapshots;
|
||||
mod stream_error_allows_next_turn;
|
||||
mod stream_no_completed;
|
||||
mod subagent_notifications;
|
||||
|
||||
@@ -10,7 +10,12 @@ use codex_history::RolloutItem;
|
||||
use codex_history::RolloutLine;
|
||||
use codex_login::CodexAuth;
|
||||
use codex_protocol::config_types::ApprovalsReviewer;
|
||||
use codex_protocol::config_types::CollaborationMode;
|
||||
use codex_protocol::config_types::ModeKind;
|
||||
use codex_protocol::config_types::Personality;
|
||||
use codex_protocol::config_types::ReasoningSummary;
|
||||
use codex_protocol::config_types::ServiceTier;
|
||||
use codex_protocol::config_types::Settings;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::PermissionProfile;
|
||||
@@ -627,6 +632,10 @@ async fn review_uses_updated_turn_permissions_and_approval_policy() {
|
||||
config.review_model = Some("gpt-5.4".to_string());
|
||||
config.model_context_window = Some(128_000);
|
||||
config.service_tier = Some(ServiceTier::Fast.request_value().to_string());
|
||||
config
|
||||
.features
|
||||
.enable(Feature::FastMode)
|
||||
.expect("enable FastMode");
|
||||
config
|
||||
.features
|
||||
.enable(Feature::TokenBudget)
|
||||
@@ -671,13 +680,22 @@ async fn review_uses_updated_turn_permissions_and_approval_policy() {
|
||||
approval_policy: Some(AskForApproval::Never),
|
||||
approvals_reviewer: Some(ApprovalsReviewer::User),
|
||||
permission_profile: Some(PermissionProfile::Disabled),
|
||||
effort: Some(Some(ReasoningEffort::XHigh)),
|
||||
personality: Some(Personality::Friendly),
|
||||
collaboration_mode: Some(CollaborationMode {
|
||||
mode: ModeKind::Plan,
|
||||
settings: Settings {
|
||||
model: "gpt-5.2".to_string(),
|
||||
reasoning_effort: Some(ReasoningEffort::XHigh),
|
||||
developer_instructions: Some("Parent planning instructions".to_string()),
|
||||
},
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("updated thread permissions should be accepted");
|
||||
|
||||
let stored_settings = codex.thread_settings_snapshot().await;
|
||||
codex
|
||||
.submit(Op::Review {
|
||||
review_request: ReviewRequest {
|
||||
@@ -691,6 +709,7 @@ async fn review_uses_updated_turn_permissions_and_approval_policy() {
|
||||
.expect("review should start");
|
||||
wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
|
||||
|
||||
assert_eq!(codex.thread_settings_snapshot().await, stored_settings);
|
||||
let request = request_log.single_request();
|
||||
assert_eq!(request.body_json()["reasoning"]["effort"], "medium");
|
||||
assert_eq!(
|
||||
@@ -751,23 +770,166 @@ async fn review_uses_updated_turn_permissions_and_approval_policy() {
|
||||
})
|
||||
.expect("review rollout should contain session metadata");
|
||||
assert_eq!(review_session_cwd, updated_cwd.as_path());
|
||||
let review_approvals_reviewer = review_rollout
|
||||
let review_context = review_rollout
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
let rollout_line: RolloutLine =
|
||||
serde_json::from_str(line).expect("review rollout line should be valid");
|
||||
match rollout_line.item {
|
||||
RolloutItem::TurnContext(turn_context) => turn_context.approvals_reviewer,
|
||||
RolloutItem::TurnContext(turn_context) => Some(turn_context),
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
.next_back();
|
||||
assert_eq!(review_approvals_reviewer, Some(ApprovalsReviewer::User));
|
||||
.next_back()
|
||||
.expect("review rollout should contain turn context");
|
||||
assert_eq!(
|
||||
review_context.approvals_reviewer,
|
||||
Some(ApprovalsReviewer::User)
|
||||
);
|
||||
assert_eq!(review_context.personality, Some(Personality::Friendly));
|
||||
// The review delegate still starts in its own default mode, not the parent's Plan mode.
|
||||
assert_eq!(
|
||||
review_context.collaboration_mode,
|
||||
Some(CollaborationMode {
|
||||
mode: ModeKind::Default,
|
||||
settings: Settings {
|
||||
model: "gpt-5.4".to_string(),
|
||||
reasoning_effort: Some(ReasoningEffort::Medium),
|
||||
developer_instructions: None,
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
let _codex_home_guard = codex_home;
|
||||
server.verify().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn review_omits_retained_tier_when_fast_mode_disabled() -> anyhow::Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let (server, request_log) =
|
||||
start_responses_server_with_sse(completed_sse(), /*expected_requests*/ 1).await;
|
||||
let test = test_codex()
|
||||
.with_model_info_override("gpt-5.4", |model| {
|
||||
model.service_tiers = vec![ModelServiceTier {
|
||||
id: ServiceTier::Flex.request_value().to_string(),
|
||||
name: "Flex".to_string(),
|
||||
description: "Flexible processing".to_string(),
|
||||
}];
|
||||
})
|
||||
.with_config(|config| {
|
||||
config
|
||||
.features
|
||||
.disable(Feature::FastMode)
|
||||
.expect("disable FastMode");
|
||||
})
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
core_test_support::submit_thread_settings(
|
||||
&test.codex,
|
||||
ThreadSettingsOverrides {
|
||||
service_tier: Some(Some(ServiceTier::Flex.request_value().to_string())),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
test.codex
|
||||
.submit(Op::Review {
|
||||
review_request: ReviewRequest {
|
||||
target: ReviewTarget::Custom {
|
||||
instructions: "review the changes".to_string(),
|
||||
},
|
||||
user_facing_hint: None,
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
test.codex
|
||||
.thread_settings_snapshot()
|
||||
.await
|
||||
.service_tier
|
||||
.as_deref(),
|
||||
Some("flex")
|
||||
);
|
||||
assert_eq!(
|
||||
request_log.single_request().body_json().get("service_tier"),
|
||||
None
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn review_resolves_inherited_summary_preferences() -> anyhow::Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let (server, request_log) =
|
||||
start_responses_server_with_sse(completed_sse(), /*expected_requests*/ 2).await;
|
||||
let test = test_codex()
|
||||
.with_model_info_override("gpt-5.2", |model| {
|
||||
model.default_reasoning_summary = ReasoningSummary::Auto;
|
||||
})
|
||||
.with_model_info_override("gpt-5.4", |model| {
|
||||
model.default_reasoning_summary = ReasoningSummary::Detailed;
|
||||
})
|
||||
.with_model("gpt-5.2")
|
||||
.with_config(|config| {
|
||||
config.review_model = Some("gpt-5.4".to_string());
|
||||
config.model_reasoning_summary = None;
|
||||
})
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
|
||||
// First follow the review model's default, then use a preference updated on the thread.
|
||||
for summary in [None, Some(ReasoningSummary::Concise)] {
|
||||
if let Some(summary) = summary {
|
||||
core_test_support::submit_thread_settings(
|
||||
&test.codex,
|
||||
ThreadSettingsOverrides {
|
||||
summary: Some(summary),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let stored_settings = test.codex.thread_settings_snapshot().await;
|
||||
test.codex
|
||||
.submit(Op::Review {
|
||||
review_request: ReviewRequest {
|
||||
target: ReviewTarget::Custom {
|
||||
instructions: "review the changes".to_string(),
|
||||
},
|
||||
user_facing_hint: None,
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
assert_eq!(test.codex.thread_settings_snapshot().await, stored_settings);
|
||||
}
|
||||
let actual = request_log
|
||||
.requests()
|
||||
.iter()
|
||||
.map(|request| {
|
||||
let body = request.body_json();
|
||||
serde_json::json!([body["model"], body["reasoning"]["summary"]])
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
actual,
|
||||
vec![
|
||||
serde_json::json!(["gpt-5.4", "detailed"]),
|
||||
serde_json::json!(["gpt-5.4", "concise"]),
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Ensure that when a custom `review_model` is set in the config, the review
|
||||
/// request uses that model (and not the main chat model).
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
use anyhow::Result;
|
||||
use codex_core::TurnInputRequest;
|
||||
use codex_features::Feature;
|
||||
use codex_models_manager::bundled_models_response;
|
||||
use codex_protocol::config_types::ReasoningSummary;
|
||||
use codex_protocol::config_types::ServiceTier;
|
||||
use codex_protocol::openai_models::ModelsResponse;
|
||||
use codex_protocol::openai_models::ReasoningEffort;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::Op;
|
||||
use codex_protocol::protocol::ThreadSettingsOverrides;
|
||||
use codex_protocol::request_user_input::RequestUserInputAnswer;
|
||||
use codex_protocol::request_user_input::RequestUserInputResponse;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use core_test_support::responses;
|
||||
use core_test_support::skip_if_no_network;
|
||||
use core_test_support::submit_thread_settings;
|
||||
use core_test_support::test_codex::test_codex;
|
||||
use core_test_support::wait_for_event;
|
||||
use core_test_support::wait_for_event_match;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn ordinary_settings_updates_keep_the_running_turn_snapshot() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let server = responses::start_mock_server().await;
|
||||
let response_mock = responses::mount_sse_sequence(
|
||||
&server,
|
||||
vec![
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("response-1"),
|
||||
responses::ev_function_call(
|
||||
"pause",
|
||||
"request_user_input",
|
||||
&json!({ "questions": [{
|
||||
"id": "continue",
|
||||
"header": "Continue",
|
||||
"question": "Continue after the settings update?",
|
||||
"options": [
|
||||
{ "label": "Yes", "description": "Continue the turn." },
|
||||
{ "label": "No", "description": "Stop the turn." }
|
||||
]
|
||||
}] })
|
||||
.to_string(),
|
||||
),
|
||||
responses::ev_completed("response-1"),
|
||||
]),
|
||||
responses::sse_completed("response-2"),
|
||||
responses::sse_completed("response-3"),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
let test = test_codex()
|
||||
.with_model("snapshot-model-a")
|
||||
.with_config(|config| {
|
||||
// This regression exercises ordinary turn construction without
|
||||
// enabling the diagnostic active-settings path.
|
||||
config
|
||||
.features
|
||||
.enable(Feature::DefaultModeRequestUserInput)
|
||||
.unwrap();
|
||||
config.features.enable(Feature::FastMode).unwrap();
|
||||
let template = bundled_models_response()
|
||||
.unwrap()
|
||||
.models
|
||||
.into_iter()
|
||||
.find(|model| model.slug == "gpt-5.4")
|
||||
.unwrap();
|
||||
config.model_catalog = Some(ModelsResponse {
|
||||
models: ["snapshot-model-a", "snapshot-model-b"]
|
||||
.into_iter()
|
||||
.map(|slug| {
|
||||
let mut model = template.clone();
|
||||
model.slug = slug.to_string();
|
||||
model.default_reasoning_summary = ReasoningSummary::Concise;
|
||||
model
|
||||
})
|
||||
.collect(),
|
||||
});
|
||||
config.model_reasoning_effort = Some(ReasoningEffort::Low);
|
||||
config.model_reasoning_summary = None;
|
||||
config.service_tier = None;
|
||||
})
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
test.codex
|
||||
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
|
||||
text: "pause before continuing".into(),
|
||||
text_elements: Vec::new(),
|
||||
}]))
|
||||
.await?;
|
||||
let paused = wait_for_event_match(&test.codex, |event| match event {
|
||||
EventMsg::RequestUserInput(request) => Some(request.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.await;
|
||||
|
||||
let mut expected = test.codex.thread_settings_snapshot().await;
|
||||
submit_thread_settings(
|
||||
&test.codex,
|
||||
ThreadSettingsOverrides {
|
||||
model: Some("snapshot-model-b".to_string()),
|
||||
effort: Some(Some(ReasoningEffort::High)),
|
||||
summary: Some(ReasoningSummary::Detailed),
|
||||
service_tier: Some(Some(ServiceTier::Fast.request_value().to_string())),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
expected.model = "snapshot-model-b".to_string();
|
||||
expected.reasoning_effort = Some(ReasoningEffort::High);
|
||||
expected.reasoning_summary = Some(ReasoningSummary::Detailed);
|
||||
expected.service_tier = Some(ServiceTier::Fast.request_value().to_string());
|
||||
expected.collaboration_mode = expected.collaboration_mode.with_updates(
|
||||
Some(expected.model.clone()),
|
||||
Some(expected.reasoning_effort.clone()),
|
||||
/*developer_instructions*/ None,
|
||||
);
|
||||
assert_eq!(test.codex.thread_settings_snapshot().await, expected);
|
||||
test.codex
|
||||
.submit(Op::UserInputAnswer {
|
||||
id: paused.turn_id.clone(),
|
||||
response: RequestUserInputResponse {
|
||||
answers: HashMap::from([(
|
||||
"continue".to_string(),
|
||||
RequestUserInputAnswer {
|
||||
answers: vec!["Yes".to_string()],
|
||||
},
|
||||
)]),
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
test.submit_text_turn("start the next turn").await?;
|
||||
|
||||
let requests = response_mock.requests();
|
||||
let actual = requests
|
||||
.iter()
|
||||
.map(|request| {
|
||||
let body = request.body_json();
|
||||
json!({
|
||||
"model": body["model"],
|
||||
"reasoning": body["reasoning"],
|
||||
"service_tier": body.get("service_tier"),
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let original = json!({
|
||||
"model": "snapshot-model-a",
|
||||
"reasoning": { "effort": "low", "summary": "concise" },
|
||||
"service_tier": null,
|
||||
});
|
||||
assert_eq!(
|
||||
actual,
|
||||
vec![
|
||||
original.clone(),
|
||||
original,
|
||||
json!({
|
||||
"model": "snapshot-model-b",
|
||||
"reasoning": { "effort": "high", "summary": "detailed" },
|
||||
"service_tier": "priority",
|
||||
})
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Reference in new issue
Block a user