From 9e301c8c9a38a867ec16395e89c1d2e7c514850b Mon Sep 17 00:00:00 2001 From: knittel-openai <153027209+knittel-openai@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:51:10 +0000 Subject: [PATCH] Add configurable Responses API request metadata (#37895) ## What changed - Add `responses_api_metadata` for product-owned key/value metadata included in every Responses API turn metadata payload, including parent and subagent requests. - Limit the map to 16 entries, ASCII identifier keys of at most 64 bytes, and values of at most 128 bytes. Reject reserved Codex metadata keys and ignore this setting in project-local configuration. - Give configured product metadata precedence over app-server client metadata while keeping it out of metadata sent to external MCP servers. ## Testing - Cover reserved-key validation, metadata precedence, MCP isolation, and propagation to parent and subagent Responses API requests. GitOrigin-RevId: a7be798294fde25145ab375a468321bb4e4a49f1 --- codex-rs/config/src/config_toml.rs | 3 ++ codex-rs/config/src/loader/mod.rs | 1 + codex-rs/core/config.schema.json | 7 +++ .../core/src/config/config_loader_tests.rs | 2 + codex-rs/core/src/config/mod.rs | 10 +++++ codex-rs/core/src/responses_metadata.rs | 41 +++++++++++++++-- codex-rs/core/src/session/turn_context.rs | 2 + codex-rs/core/src/turn_metadata.rs | 44 ++++++++++++++++--- codex-rs/core/src/turn_metadata_tests.rs | 31 +++++++++++++ .../suite/responses_api_proxy_headers.rs | 13 +++++- codex-rs/thread-manager-sample/src/main.rs | 1 + 11 files changed, 144 insertions(+), 11 deletions(-) diff --git a/codex-rs/config/src/config_toml.rs b/codex-rs/config/src/config_toml.rs index 821026255d..fd888debfe 100644 --- a/codex-rs/config/src/config_toml.rs +++ b/codex-rs/config/src/config_toml.rs @@ -369,6 +369,9 @@ pub struct ConfigToml { /// Optional product SKU forwarded on host-owned Codex Apps MCP requests. pub apps_mcp_product_sku: Option, + /// Bounded, product-owned metadata attached to every Responses API request. + pub responses_api_metadata: Option>, + /// Orchestrator-owned feature settings. pub orchestrator: Option, diff --git a/codex-rs/config/src/loader/mod.rs b/codex-rs/config/src/loader/mod.rs index cc661fc8d5..ceae7853ca 100644 --- a/codex-rs/config/src/loader/mod.rs +++ b/codex-rs/config/src/loader/mod.rs @@ -71,6 +71,7 @@ const PROJECT_LOCAL_CONFIG_DENYLIST: &[&str] = &[ "openai_base_url", "chatgpt_base_url", "apps_mcp_product_sku", + "responses_api_metadata", "model_provider", "model_providers", "notify", diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index c94db1eb96..f9a046df44 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -5819,6 +5819,13 @@ "default": null, "description": "Experimental / do not use. Realtime websocket session selection. `version` controls v1/v2 and `type` controls conversational/transcription." }, + "responses_api_metadata": { + "additionalProperties": { + "type": "string" + }, + "description": "Bounded, product-owned metadata attached to every Responses API request.", + "type": "object" + }, "review_model": { "description": "Review model override used by the `/review` feature.", "type": "string" diff --git a/codex-rs/core/src/config/config_loader_tests.rs b/codex-rs/core/src/config/config_loader_tests.rs index 65ad3d0ffc..0793850ac1 100644 --- a/codex-rs/core/src/config/config_loader_tests.rs +++ b/codex-rs/core/src/config/config_loader_tests.rs @@ -3372,6 +3372,7 @@ model_instructions_file = "instructions.md" openai_base_url = "https://attacker.example/v1" chatgpt_base_url = "https://attacker.example/backend-api" apps_mcp_product_sku = "attacker" +responses_api_metadata = { codex_security_surface = "attacker" } model_provider = "attacker" notify = ["sh", "-c", "echo attacker"] profile = "attacker" @@ -3425,6 +3426,7 @@ wire_api = "responses" "openai_base_url", "chatgpt_base_url", "apps_mcp_product_sku", + "responses_api_metadata", "model_provider", "model_providers", "notify", diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 5ca2239e30..6463032282 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -145,6 +145,7 @@ use crate::config::permissions::validate_user_permission_profile_names; use crate::config_lock::config_without_lock_controls; use crate::config_lock::lock_layer_from_config; use crate::config_lock::read_config_lock_from_path; +use crate::responses_metadata::validate_extra_metadata; use codex_network_proxy::NetworkProxyConfig; use toml::Value as TomlValue; use toml_edit::DocumentMut; @@ -1000,6 +1001,9 @@ pub struct Config { /// Optional product SKU forwarded to the host-owned apps MCP server. pub apps_mcp_product_sku: Option, + /// Bounded, product-owned metadata attached to every Responses API request. + pub responses_api_metadata: BTreeMap, + /// Machine-local realtime audio device preferences used by realtime voice. pub realtime_audio: RealtimeAudioConfig, @@ -3245,6 +3249,11 @@ impl Config { validate_model_providers(&cfg.model_providers) .map_err(|message| std::io::Error::new(std::io::ErrorKind::InvalidInput, message))?; + if let Some(responses_api_metadata) = cfg.responses_api_metadata.as_ref() { + validate_extra_metadata(responses_api_metadata.iter()).map_err(|message| { + std::io::Error::new(std::io::ErrorKind::InvalidInput, message) + })?; + } let orchestrator = cfg.orchestrator.as_ref(); let orchestrator_skills_enabled = resolve_orchestrator_feature_enabled(orchestrator.and_then(|value| value.skills.as_ref())); @@ -4260,6 +4269,7 @@ impl Config { respect_system_proxy, psp: psp.unwrap_or_default(), apps_mcp_product_sku: cfg.apps_mcp_product_sku.clone(), + responses_api_metadata: cfg.responses_api_metadata.unwrap_or_default(), realtime_audio: cfg .audio .map_or_else(RealtimeAudioConfig::default, |audio| RealtimeAudioConfig { diff --git a/codex-rs/core/src/responses_metadata.rs b/codex-rs/core/src/responses_metadata.rs index d2f05011f6..c1f9e1e6b0 100644 --- a/codex-rs/core/src/responses_metadata.rs +++ b/codex-rs/core/src/responses_metadata.rs @@ -71,6 +71,9 @@ const RESERVED_METADATA_KEYS: &[&str] = &[ SANDBOX_MODE_KEY, WORKSPACES_KEY, ]; +const MAX_EXTRA_METADATA_ENTRIES: usize = 16; +const MAX_EXTRA_METADATA_KEY_BYTES: usize = 64; +const MAX_EXTRA_METADATA_VALUE_BYTES: usize = 128; /// Metadata attached to model requests whose purpose is conversation compaction. /// @@ -348,9 +351,9 @@ impl CodexResponsesMetadata { tool_namespaces_info: self.tool_namespaces_info.as_ref(), turn_started_at_unix_ms: self.turn_started_at_unix_ms, compaction, - // responsesapi_client_metadata enriches the Codex turn metadata blob, not literal - // top-level Responses client_metadata. Reserved Codex-owned keys are filtered when - // these extras enter turn state. + // Extra metadata enriches the Codex turn metadata blob, not literal top-level + // Responses client_metadata. Product metadata is validated while loading config; + // app-server metadata has reserved Codex-owned keys filtered when it enters turn state. extra: &self.extra, } } @@ -396,13 +399,43 @@ fn insert_header(headers: &mut ApiHeaderMap, name: &'static str, value: &str) { } } -pub(crate) fn filter_extra_metadata(extra: HashMap) -> BTreeMap { +pub(crate) fn validate_extra_metadata<'a>( + extra: impl IntoIterator, +) -> Result<(), &'static str> { + let mut count = 0; + for (key, value) in extra { + count += 1; + if count > MAX_EXTRA_METADATA_ENTRIES { + return Err("responses_api_metadata may contain at most 16 entries"); + } + if key.len() > MAX_EXTRA_METADATA_KEY_BYTES || !valid_extra_metadata_key(key) { + return Err("responses_api_metadata keys must be short ASCII identifiers"); + } + if RESERVED_METADATA_KEYS.contains(&key.as_str()) { + return Err("responses_api_metadata contains a reserved key"); + } + if value.len() > MAX_EXTRA_METADATA_VALUE_BYTES { + return Err("responses_api_metadata values may contain at most 128 bytes"); + } + } + Ok(()) +} + +pub(crate) fn filter_extra_metadata( + extra: impl IntoIterator, +) -> BTreeMap { extra .into_iter() .filter(|(key, _)| !RESERVED_METADATA_KEYS.contains(&key.as_str())) .collect() } +fn valid_extra_metadata_key(key: &str) -> bool { + let mut bytes = key.bytes(); + bytes.next().is_some_and(|byte| byte.is_ascii_alphabetic()) + && bytes.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-')) +} + fn non_empty_workspaces( workspaces: &BTreeMap, ) -> Option<&BTreeMap> { diff --git a/codex-rs/core/src/session/turn_context.rs b/codex-rs/core/src/session/turn_context.rs index 82c5a4cbe5..bd2d09292e 100644 --- a/codex-rs/core/src/session/turn_context.rs +++ b/codex-rs/core/src/session/turn_context.rs @@ -581,6 +581,8 @@ impl Session { session_configuration.windows_sandbox_level, network.is_some(), )); + turn_metadata_state + .set_responses_api_metadata(per_turn_config.responses_api_metadata.clone()); let (current_date, timezone) = local_time_context(); let extension_data = Arc::new(codex_extension_api::ExtensionData::new(sub_id.clone())); extension_data.insert(skills_snapshot); diff --git a/codex-rs/core/src/turn_metadata.rs b/codex-rs/core/src/turn_metadata.rs index 71242a8655..7dfbce31cc 100644 --- a/codex-rs/core/src/turn_metadata.rs +++ b/codex-rs/core/src/turn_metadata.rs @@ -110,6 +110,7 @@ pub(crate) struct TurnMetadataState { enriched_workspaces: RwLock>>, tool_namespaces_info: RwLock>, turn_started_at_unix_ms: RwLock>, + responses_api_metadata: RwLock>, responsesapi_client_metadata: RwLock>, user_input_requested_during_turn: AtomicBool, enrichment_task: Mutex>>, @@ -158,6 +159,7 @@ impl TurnMetadataState { enriched_workspaces: RwLock::new(None), tool_namespaces_info: RwLock::new(None), turn_started_at_unix_ms: RwLock::new(None), + responses_api_metadata: RwLock::new(BTreeMap::new()), responsesapi_client_metadata: RwLock::new(BTreeMap::new()), user_input_requested_during_turn: AtomicBool::new(false), enrichment_task: Mutex::new(None), @@ -168,7 +170,7 @@ impl TurnMetadataState { &self, context: McpTurnMetadataContext<'_>, ) -> Option { - let mut responses_metadata = self.responses_metadata_template(); + let mut responses_metadata = self.mcp_metadata_template(); // Never serialize harness-owned tool inventory for external MCP servers. responses_metadata.tool_namespaces_info = None; let Value::Object(mut metadata) = responses_metadata.turn_metadata_value()? else { @@ -249,6 +251,16 @@ impl TurnMetadataState { filter_extra_metadata(responsesapi_client_metadata); } + pub(crate) fn set_responses_api_metadata( + &self, + responses_api_metadata: BTreeMap, + ) { + *self + .responses_api_metadata + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = responses_api_metadata; + } + pub(crate) fn workspace_kind(&self) -> Option { self.responsesapi_client_metadata .read() @@ -258,6 +270,30 @@ impl TurnMetadataState { } fn responses_metadata_template(&self) -> CodexResponsesMetadata { + let mut metadata = self.mcp_metadata_template(); + metadata.extra.extend( + self.responses_api_metadata + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(), + ); + metadata + } + + fn mcp_metadata_template(&self) -> CodexResponsesMetadata { + let mut extra = self + .responsesapi_client_metadata + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + for key in self + .responses_api_metadata + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .keys() + { + extra.remove(key); + } CodexResponsesMetadata { turn_id: Some(self.turn_id.clone()), forked_from_thread_id: self.forked_from_thread_id, @@ -275,11 +311,7 @@ impl TurnMetadataState { .unwrap_or_else(std::sync::PoisonError::into_inner) .clone(), turn_started_at_unix_ms: self.current_turn_started_at_unix_ms(), - extra: self - .responsesapi_client_metadata - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .clone(), + extra, ..CodexResponsesMetadata::new( String::new(), self.session_id.clone(), diff --git a/codex-rs/core/src/turn_metadata_tests.rs b/codex-rs/core/src/turn_metadata_tests.rs index 8287b299cb..4346983569 100644 --- a/codex-rs/core/src/turn_metadata_tests.rs +++ b/codex-rs/core/src/turn_metadata_tests.rs @@ -11,6 +11,7 @@ use crate::responses_metadata::TurnToolFunctionInfo; use crate::responses_metadata::TurnToolNamespaceInfo; use crate::responses_metadata::TurnToolSource; use crate::responses_metadata::WINDOW_ID_KEY; +use crate::responses_metadata::validate_extra_metadata; use crate::sandbox_tags::permission_profile_sandbox_tag; use codex_analytics::CompactionImplementation; use codex_analytics::CompactionPhase; @@ -630,8 +631,16 @@ fn turn_metadata_state_merges_client_metadata_without_replacing_reserved_fields( WindowsSandboxLevel::Disabled, /*enforce_managed_network*/ false, ); + state.set_responses_api_metadata(BTreeMap::from([( + "codex_security_surface".to_string(), + "sdk".to_string(), + )])); state.set_parent_turn_id("parent-turn-a".to_string()); state.set_responsesapi_client_metadata(HashMap::from([ + ( + "codex_security_surface".to_string(), + "client-supplied".to_string(), + ), ("fiber_run_id".to_string(), "fiber-123".to_string()), ("origin".to_string(), "東京".to_string()), ("workspace_kind".to_string(), "projectless".to_string()), @@ -710,6 +719,7 @@ fn turn_metadata_state_merges_client_metadata_without_replacing_reserved_fields( assert_eq!(json["fiber_run_id"].as_str(), Some("fiber-123")); assert_eq!(json["origin"].as_str(), Some("東京")); assert_eq!(json["workspace_kind"].as_str(), Some("projectless")); + assert_eq!(json["codex_security_surface"].as_str(), Some("sdk")); assert_eq!(json["model"].as_str(), Some("client-supplied")); assert_eq!(json["reasoning_effort"].as_str(), Some("client-supplied")); assert_eq!(json["session_id"].as_str(), Some("session-a")); @@ -766,6 +776,10 @@ fn turn_metadata_state_merges_client_metadata_without_replacing_reserved_fields( model_request_json["thread_source"].as_str(), Some("automation") ); + assert_eq!( + model_request_json["codex_security_surface"].as_str(), + Some("sdk") + ); assert_eq!( model_request_json[INSTALLATION_ID_KEY].as_str(), Some("installation-a") @@ -810,6 +824,7 @@ fn turn_metadata_state_merges_client_metadata_without_replacing_reserved_fields( assert!(meta.get(TOOL_NAMESPACES_INFO_KEY).is_none()); assert!(meta.get(PARENT_TURN_ID_KEY).is_none()); assert!(meta.get(WINDOW_ID_KEY).is_none()); + assert!(meta.get("codex_security_surface").is_none()); assert_eq!(state.workspace_kind().as_deref(), Some("projectless")); } @@ -831,6 +846,10 @@ fn turn_metadata_state_overlays_compaction_only_on_compaction_requests() { WindowsSandboxLevel::Disabled, /*enforce_managed_network*/ false, ); + state.set_responses_api_metadata(BTreeMap::from([( + "codex_security_surface".to_string(), + "sdk".to_string(), + )])); state.set_responsesapi_client_metadata(HashMap::from([( "compaction".to_string(), "client-supplied".to_string(), @@ -850,6 +869,7 @@ fn turn_metadata_state_overlays_compaction_only_on_compaction_requests() { assert_eq!(compact_json["request_kind"].as_str(), Some("compaction")); assert_eq!(compact_json["turn_id"].as_str(), Some("turn-a")); assert_eq!(compact_json[WINDOW_ID_KEY].as_str(), Some("thread-a:2")); + assert_eq!(compact_json["codex_security_surface"].as_str(), Some("sdk")); assert_eq!( compact_json["compaction"], serde_json::json!({ @@ -865,9 +885,20 @@ fn turn_metadata_state_overlays_compaction_only_on_compaction_requests() { let regular_json: Value = serde_json::from_str(®ular_header).expect("json"); assert_eq!(regular_json["request_kind"].as_str(), Some("turn")); assert_eq!(regular_json[WINDOW_ID_KEY].as_str(), Some("thread-a:3")); + assert_eq!(regular_json["codex_security_surface"].as_str(), Some("sdk")); assert!(regular_json.get("compaction").is_none()); } +#[test] +fn responses_api_metadata_rejects_reserved_keys() { + assert_eq!( + validate_extra_metadata( + BTreeMap::from([("thread_source".to_string(), "sdk".to_string())]).iter() + ), + Err("responses_api_metadata contains a reserved key") + ); +} + #[tokio::test] async fn turn_metadata_state_preserves_lineage_after_git_enrichment() { let (_temp_dir, repo_path) = create_clean_git_repo("repo").await; diff --git a/codex-rs/core/tests/suite/responses_api_proxy_headers.rs b/codex-rs/core/tests/suite/responses_api_proxy_headers.rs index 768bbf11db..acbd62fa8e 100644 --- a/codex-rs/core/tests/suite/responses_api_proxy_headers.rs +++ b/codex-rs/core/tests/suite/responses_api_proxy_headers.rs @@ -33,7 +33,8 @@ const SPAWN_CALL_ID: &str = "spawn-call-1"; const REQUEST_POLL_INTERVAL: Duration = Duration::from_millis(/*millis*/ 20); const TURN_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 60); #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn responses_api_parent_and_subagent_requests_include_identity_headers() -> Result<()> { +async fn responses_api_parent_and_subagent_requests_include_identity_headers_and_product_metadata() +-> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; @@ -90,6 +91,9 @@ async fn responses_api_parent_and_subagent_requests_include_identity_headers() - .features .disable(Feature::EnableRequestCompression) .expect("test config should allow feature update"); + config + .responses_api_metadata + .insert("codex_security_surface".to_string(), "sdk".to_string()); }); let test = builder.build(&server).await?; submit_turn_with_timeout(&test, PARENT_PROMPT).await?; @@ -126,6 +130,12 @@ async fn responses_api_parent_and_subagent_requests_include_identity_headers() - child.header("x-codex-parent-thread-id").as_deref(), Some(parent_thread_id) ); + let parent_turn_metadata: serde_json::Value = serde_json::from_str( + &parent + .header("x-codex-turn-metadata") + .ok_or_else(|| anyhow!("parent request missing x-codex-turn-metadata"))?, + )?; + assert_eq!(parent_turn_metadata["codex_security_surface"], json!("sdk")); let child_turn_metadata: serde_json::Value = serde_json::from_str( &child .header("x-codex-turn-metadata") @@ -136,6 +146,7 @@ async fn responses_api_parent_and_subagent_requests_include_identity_headers() - child_turn_metadata["parent_thread_id"].as_str(), Some(parent_thread_id) ); + assert_eq!(child_turn_metadata["codex_security_surface"], json!("sdk")); Ok(()) } diff --git a/codex-rs/thread-manager-sample/src/main.rs b/codex-rs/thread-manager-sample/src/main.rs index 6dbc6d5214..d36508ce55 100644 --- a/codex-rs/thread-manager-sample/src/main.rs +++ b/codex-rs/thread-manager-sample/src/main.rs @@ -281,6 +281,7 @@ fn new_config(model: Option, arg0_paths: Arg0DispatchPaths) -> anyhow::R chatgpt_base_url: "https://chatgpt.com/backend-api/".to_string(), respect_system_proxy: false, apps_mcp_product_sku: None, + responses_api_metadata: BTreeMap::new(), realtime_audio: RealtimeAudioConfig::default(), experimental_realtime_ws_base_url: None, experimental_realtime_webrtc_call_base_url: None,