feat: migrate to new constraint-based loading strategy (#8251)

This is a significant change to how layers of configuration are applied.
In particular, the `ConfigLayerStack` now has two important fields:

- `layers: Vec<ConfigLayerEntry>`
- `requirements: ConfigRequirements`

We merge `TomlValue`s across the layers, but they are subject to
`ConfigRequirements` before creating a `Config`.

How I would review this PR:

- start with `codex-rs/app-server-protocol/src/protocol/v2.rs` and note
the new variants added to the `ConfigLayerSource` enum:
`LegacyManagedConfigTomlFromFile` and `LegacyManagedConfigTomlFromMdm`
- note that `ConfigLayerSource` now has a `precedence()` method and
implements `PartialOrd`
- `codex-rs/core/src/config_loader/layer_io.rs` is responsible for
loading "admin" preferences from `/etc/codex/managed_config.toml` and
MDM. Because `/etc/codex/managed_config.toml` is now deprecated in favor
of `/etc/codex/requirements.toml` and `/etc/codex/config.toml`, we now
include some extra information on the `LoadedConfigLayers` returned in
`layer_io.rs`.
- `codex-rs/core/src/config_loader/mod.rs` has major changes to
`load_config_layers_state()`, which is what produces `ConfigLayerStack`.
The docstring has the new specification and describes the various layers
that will be loaded and the precedence order.
- It uses the information from `LoaderOverrides` "twice," both in the
spirit of legacy support:
- We use one instances to derive an instance of `ConfigRequirements`.
Currently, the only field in `managed_config.toml` that contributes to
`ConfigRequirements` is `approval_policy`. This PR introduces
`Constrained::allow_only()` to support this.
- We use a clone of `LoaderOverrides` to derive
`ConfigLayerSource::LegacyManagedConfigTomlFromFile` and
`ConfigLayerSource::LegacyManagedConfigTomlFromMdm` layers, as
appropriate. As before, this ends up being a "best effort" at enterprise
controls, but is enforcement is not guaranteed like it is for
`ConfigRequirements`.
- Now we only create a "user" layer if `$CODEX_HOME/config.toml` exists.
(Previously, a user layer was always created for `ConfigLayerStack`.)
- Similarly, we only add a "session flags" layer if there are CLI
overrides.
- `config_loader/state.rs` contains the updated implementation for
`ConfigLayerStack`. Note the public API is largely the same as before,
but the implementation is quite different. We leverage the fact that
`ConfigLayerSource` is now `PartialOrd` to ensure layers are in the
correct order.
- A `Config` constructed via `ConfigBuilder.build()` will use
`load_config_layers_state()` to create the `ConfigLayerStack` and use
the associated `ConfigRequirements` when constructing the `Config`
object.
- That said, a `Config` constructed via
`Config::load_from_base_config_with_overrides()` does _not_ yet use
`ConfigBuilder`, so it creates a `ConfigRequirements::default()` instead
of loading a proper `ConfigRequirements`. I will fix this in a
subsequent PR.

Then the following files are mostly test changes:

```
codex-rs/app-server/tests/suite/v2/config_rpc.rs
codex-rs/core/src/config/service.rs
codex-rs/core/src/config_loader/tests.rs
```

Again, because we do not always include "user" and "session flags"
layers when the contents are empty, `ConfigLayerStack` sometimes has
fewer layers than before (and the precedence order changed slightly),
which is the main reason integration tests changed.

codex-rs/core/src/config/constraint.rs

@@ -19,6 +19,12 @@ impl ConstraintError {
             ),
         }
     }
+
+    pub fn empty_field(field_name: impl Into<String>) -> Self {
+        Self {
+            message: format!("field `{}` cannot be empty", field_name.into()),
+        }
+    }
 }
 
 pub type ConstraintResult<T> = Result<T, ConstraintError>;
@@ -57,6 +63,32 @@ impl<T: Send + Sync> Constrained<T> {
         }
     }
 
+    pub fn allow_only(value: T) -> Self
+    where
+        T: PartialEq + Send + Sync + fmt::Debug + Clone + 'static,
+    {
+        #[expect(clippy::expect_used)]
+        Self::new(value.clone(), move |candidate| {
+            if *candidate == value {
+                Ok(())
+            } else {
+                Err(ConstraintError::invalid_value(
+                    format!("{candidate:?}"),
+                    format!("{value:?}"),
+                ))
+            }
+        })
+        .expect("initial value should always be valid")
+    }
+
+    /// Allow any value of T, using T's Default as the initial value.
+    pub fn allow_any_from_default() -> Self
+    where
+        T: Default,
+    {
+        Self::allow_any(T::default())
+    }
+
     pub fn allow_values(initial_value: T, allowed: Vec<T>) -> ConstraintResult<Self>
     where
         T: PartialEq + Send + Sync + fmt::Debug + 'static,
@@ -129,6 +161,12 @@ mod tests {
         assert_eq!(constrained.value(), -10);
     }
 
+    #[test]
+    fn constrained_allow_any_default_uses_default_value() {
+        let constrained = Constrained::<i32>::allow_any_from_default();
+        assert_eq!(constrained.value(), 0);
+    }
+
     #[test]
     fn constrained_new_rejects_invalid_initial_value() {
         let result = Constrained::new(0, |value| {

codex-rs/core/src/config/mod.rs

@@ -12,6 +12,7 @@ use crate::config::types::ShellEnvironmentPolicy;
 use crate::config::types::ShellEnvironmentPolicyToml;
 use crate::config::types::Tui;
 use crate::config::types::UriBasedFileOpener;
+use crate::config_loader::ConfigRequirements;
 use crate::config_loader::LoaderOverrides;
 use crate::config_loader::load_config_layers_state;
 use crate::features::Feature;
@@ -356,7 +357,12 @@ impl ConfigBuilder {
         let config_toml: ConfigToml = merged_toml
             .try_into()
             .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
-        Config::load_from_base_config_with_overrides(config_toml, harness_overrides, codex_home)
+        Config::load_config_with_requirements(
+            config_toml,
+            harness_overrides,
+            codex_home,
+            config_layer_stack.requirements().clone(),
+        )
     }
 }
 
@@ -390,20 +396,18 @@ impl Config {
     }
 }
 
-/// DEPRECATED: Use [Config::load_with_cli_overrides()] instead because this
-/// codepath is not guaranteed to honor [ConfigRequirements].
+/// DEPRECATED: Use [Config::load_with_cli_overrides()] instead because working
+/// with [ConfigToml] directly means that [ConfigRequirements] have not been
+/// applied yet, which risks failing to enforce required constraints.
 pub async fn load_config_as_toml_with_cli_overrides(
     codex_home: &Path,
     cli_overrides: Vec<(String, TomlValue)>,
 ) -> std::io::Result<ConfigToml> {
-    let root_value = load_resolved_config(
-        codex_home,
-        cli_overrides,
-        crate::config_loader::LoaderOverrides::default(),
-    )
-    .await?;
+    let config_layer_stack =
+        load_config_layers_state(codex_home, &cli_overrides, LoaderOverrides::default()).await?;
 
-    let cfg = deserialize_config_toml_with_base(root_value, codex_home).map_err(|e| {
+    let merged_toml = config_layer_stack.effective_config();
+    let cfg = deserialize_config_toml_with_base(merged_toml, codex_home).map_err(|e| {
         tracing::error!("Failed to deserialize overridden config: {e}");
         e
     })?;
@@ -411,15 +415,6 @@ pub async fn load_config_as_toml_with_cli_overrides(
     Ok(cfg)
 }
 
-async fn load_resolved_config(
-    codex_home: &Path,
-    cli_overrides: Vec<(String, TomlValue)>,
-    overrides: crate::config_loader::LoaderOverrides,
-) -> std::io::Result<TomlValue> {
-    let layers = load_config_layers_state(codex_home, &cli_overrides, overrides).await?;
-    Ok(layers.effective_config())
-}
-
 fn deserialize_config_toml_with_base(
     root_value: TomlValue,
     config_base_dir: &Path,
@@ -435,13 +430,18 @@ fn deserialize_config_toml_with_base(
 pub async fn load_global_mcp_servers(
     codex_home: &Path,
 ) -> std::io::Result<BTreeMap<String, McpServerConfig>> {
-    let root_value = load_resolved_config(
-        codex_home,
-        Vec::new(),
-        crate::config_loader::LoaderOverrides::default(),
-    )
-    .await?;
-    let Some(servers_value) = root_value.get("mcp_servers") else {
+    // In general, Config::load_with_cli_overrides() should be used to load the
+    // full config with requirements.toml applied, but in this case, we need
+    // access to the raw TOML in order to warn the user about deprecated fields.
+    //
+    // Note that a more precise way to do this would be to audit the individual
+    // config layers for deprecated fields rather than reporting on the merged
+    // result.
+    let cli_overrides = Vec::<(String, TomlValue)>::new();
+    let config_layer_stack =
+        load_config_layers_state(codex_home, &cli_overrides, LoaderOverrides::default()).await?;
+    let merged_toml = config_layer_stack.effective_config();
+    let Some(servers_value) = merged_toml.get("mcp_servers") else {
         return Ok(BTreeMap::new());
     };
 
@@ -999,6 +999,16 @@ impl Config {
         cfg: ConfigToml,
         overrides: ConfigOverrides,
         codex_home: PathBuf,
+    ) -> std::io::Result<Self> {
+        let requirements = ConfigRequirements::default();
+        Self::load_config_with_requirements(cfg, overrides, codex_home, requirements)
+    }
+
+    fn load_config_with_requirements(
+        cfg: ConfigToml,
+        overrides: ConfigOverrides,
+        codex_home: PathBuf,
+        requirements: ConfigRequirements,
     ) -> std::io::Result<Self> {
         let user_instructions = Self::load_instructions(Some(&codex_home));
 
@@ -1104,7 +1114,8 @@ impl Config {
                     AskForApproval::default()
                 }
             });
-        let approval_policy = Constrained::allow_any(approval_policy);
+        // TODO(dylan): We should be able to leverage ConfigLayerStack so that
+        // we can reliably check this at every config level.
         let did_user_set_custom_approval_policy_or_sandbox_mode = approval_policy_override
             .is_some()
             || config_profile.approval_policy.is_some()
@@ -1221,6 +1232,16 @@ impl Config {
 
         let check_for_update_on_startup = cfg.check_for_update_on_startup.unwrap_or(true);
 
+        // Ensure that every field of ConfigRequirements is applied to the final
+        // Config.
+        let ConfigRequirements {
+            approval_policy: mut constrained_approval_policy,
+        } = requirements;
+
+        constrained_approval_policy
+            .set(approval_policy)
+            .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, format!("{e}")))?;
+
         let config = Self {
             model,
             review_model,
@@ -1229,7 +1250,7 @@ impl Config {
             model_provider_id,
             model_provider,
             cwd: resolved_cwd,
-            approval_policy,
+            approval_policy: constrained_approval_policy,
             sandbox_policy,
             did_user_set_custom_approval_policy_or_sandbox_mode,
             forced_auto_mode_downgraded_on_windows,
@@ -1920,18 +1941,22 @@ trust_level = "trusted"
         std::fs::write(&config_path, "mcp_oauth_credentials_store = \"file\"\n")?;
         std::fs::write(&managed_path, "mcp_oauth_credentials_store = \"keyring\"\n")?;
 
-        let overrides = crate::config_loader::LoaderOverrides {
+        let overrides = LoaderOverrides {
             managed_config_path: Some(managed_path.clone()),
             #[cfg(target_os = "macos")]
             managed_preferences_base64: None,
         };
 
-        let root_value = load_resolved_config(codex_home.path(), Vec::new(), overrides).await?;
-        let cfg =
-            deserialize_config_toml_with_base(root_value, codex_home.path()).map_err(|e| {
-                tracing::error!("Failed to deserialize overridden config: {e}");
-                e
-            })?;
+        let config_layer_stack =
+            load_config_layers_state(codex_home.path(), &Vec::new(), overrides).await?;
+        let cfg = deserialize_config_toml_with_base(
+            config_layer_stack.effective_config(),
+            codex_home.path(),
+        )
+        .map_err(|e| {
+            tracing::error!("Failed to deserialize overridden config: {e}");
+            e
+        })?;
         assert_eq!(
             cfg.mcp_oauth_credentials_store,
             Some(OAuthCredentialsStoreMode::Keyring),
@@ -2035,24 +2060,27 @@ trust_level = "trusted"
         )?;
         std::fs::write(&managed_path, "model = \"managed_config\"\n")?;
 
-        let overrides = crate::config_loader::LoaderOverrides {
+        let overrides = LoaderOverrides {
             managed_config_path: Some(managed_path),
             #[cfg(target_os = "macos")]
             managed_preferences_base64: None,
         };
 
-        let root_value = load_resolved_config(
+        let config_layer_stack = load_config_layers_state(
             codex_home.path(),
-            vec![("model".to_string(), TomlValue::String("cli".to_string()))],
+            &[("model".to_string(), TomlValue::String("cli".to_string()))],
             overrides,
         )
         .await?;
 
-        let cfg =
-            deserialize_config_toml_with_base(root_value, codex_home.path()).map_err(|e| {
-                tracing::error!("Failed to deserialize overridden config: {e}");
-                e
-            })?;
+        let cfg = deserialize_config_toml_with_base(
+            config_layer_stack.effective_config(),
+            codex_home.path(),
+        )
+        .map_err(|e| {
+            tracing::error!("Failed to deserialize overridden config: {e}");
+            e
+        })?;
 
         assert_eq!(cfg.model.as_deref(), Some("managed_config"));
         Ok(())

codex-rs/core/src/config/service.rs

@@ -20,7 +20,9 @@ use codex_app_server_protocol::ConfigWriteResponse;
 use codex_app_server_protocol::MergeStrategy;
 use codex_app_server_protocol::OverriddenMetadata;
 use codex_app_server_protocol::WriteStatus;
+use codex_utils_absolute_path::AbsolutePathBuf;
 use serde_json::Value as JsonValue;
+use std::borrow::Cow;
 use std::path::Path;
 use std::path::PathBuf;
 use thiserror::Error;
@@ -146,7 +148,13 @@ impl ConfigService {
         Ok(ConfigReadResponse {
             config,
             origins: layers.origins(),
-            layers: params.include_layers.then(|| layers.layers_high_to_low()),
+            layers: params.include_layers.then(|| {
+                layers
+                    .layers_high_to_low()
+                    .iter()
+                    .map(|layer| layer.as_layer())
+                    .collect()
+            }),
         })
     }
 
@@ -194,11 +202,14 @@ impl ConfigService {
         expected_version: Option<String>,
         edits: Vec<(String, JsonValue, MergeStrategy)>,
     ) -> Result<ConfigWriteResponse, ConfigServiceError> {
-        let allowed_path = self.codex_home.join(CONFIG_TOML_FILE);
-        let provided_path = file_path
-            .as_ref()
-            .map(PathBuf::from)
-            .unwrap_or_else(|| allowed_path.clone());
+        let allowed_path =
+            AbsolutePathBuf::resolve_path_against_base(CONFIG_TOML_FILE, &self.codex_home)
+                .map_err(|err| ConfigServiceError::io("failed to resolve user config path", err))?;
+        let provided_path = match file_path {
+            Some(path) => AbsolutePathBuf::from_absolute_path(PathBuf::from(path))
+                .map_err(|err| ConfigServiceError::io("failed to resolve user config path", err))?,
+            None => allowed_path.clone(),
+        };
 
         if !paths_match(&allowed_path, &provided_path) {
             return Err(ConfigServiceError::write(
@@ -211,9 +222,13 @@ impl ConfigService {
             .load_layers_state()
             .await
             .map_err(|err| ConfigServiceError::io("failed to load configuration", err))?;
+        let user_layer = match layers.get_user_layer() {
+            Some(layer) => Cow::Borrowed(layer),
+            None => Cow::Owned(create_empty_user_layer(&allowed_path).await?),
+        };
 
         if let Some(expected) = expected_version.as_deref()
-            && expected != layers.user.version
+            && expected != user_layer.version
         {
             return Err(ConfigServiceError::write(
                 ConfigWriteErrorCode::ConfigVersionConflict,
@@ -221,7 +236,7 @@ impl ConfigService {
             ));
         }
 
-        let mut user_config = layers.user.config.clone();
+        let mut user_config = user_layer.config.clone();
         let mut parsed_segments = Vec::new();
         let mut config_edits = Vec::new();
 
@@ -273,7 +288,7 @@ impl ConfigService {
             )
         })?;
 
-        let updated_layers = layers.with_user_config(user_config.clone());
+        let updated_layers = layers.with_user_config(&provided_path, user_config.clone());
         let effective = updated_layers.effective_config();
         validate_config(&effective).map_err(|err| {
             ConfigServiceError::write(
@@ -296,16 +311,19 @@ impl ConfigService {
             .map(|_| WriteStatus::OkOverridden)
             .unwrap_or(WriteStatus::Ok);
 
-        let file_path = provided_path
-            .canonicalize()
-            .unwrap_or(provided_path.clone())
-            .display()
-            .to_string();
-
         Ok(ConfigWriteResponse {
             status,
-            version: updated_layers.user.version.clone(),
-            file_path,
+            version: updated_layers
+                .get_user_layer()
+                .ok_or_else(|| {
+                    ConfigServiceError::write(
+                        ConfigWriteErrorCode::UserLayerNotFound,
+                        "user layer not found in updated layers",
+                    )
+                })?
+                .version
+                .clone(),
+            file_path: provided_path,
             overridden_metadata: overridden,
         })
     }
@@ -320,6 +338,32 @@ impl ConfigService {
     }
 }
 
+async fn create_empty_user_layer(
+    config_toml: &AbsolutePathBuf,
+) -> Result<ConfigLayerEntry, ConfigServiceError> {
+    let toml_value = match tokio::fs::read_to_string(config_toml).await {
+        Ok(contents) => toml::from_str(&contents).map_err(|e| {
+            ConfigServiceError::toml("failed to parse existing user config.toml", e)
+        })?,
+        Err(e) => {
+            if e.kind() == std::io::ErrorKind::NotFound {
+                tokio::fs::write(config_toml, "").await.map_err(|e| {
+                    ConfigServiceError::io("failed to create empty user config.toml", e)
+                })?;
+                TomlValue::Table(toml::map::Map::new())
+            } else {
+                return Err(ConfigServiceError::io("failed to read user config.toml", e));
+            }
+        }
+    };
+    Ok(ConfigLayerEntry::new(
+        ConfigLayerSource::User {
+            file: config_toml.clone(),
+        },
+        toml_value,
+    ))
+}
+
 fn parse_value(value: JsonValue) -> Result<Option<TomlValue>, String> {
     if value.is_null() {
         return Ok(None);
@@ -501,10 +545,25 @@ fn value_at_path<'a>(root: &'a TomlValue, segments: &[String]) -> Option<&'a Tom
 
 fn override_message(layer: &ConfigLayerSource) -> String {
     match layer {
-        ConfigLayerSource::Mdm { .. } => "Overridden by managed policy (mdm)".to_string(),
-        ConfigLayerSource::System { .. } => "Overridden by managed config (system)".to_string(),
+        ConfigLayerSource::Mdm { domain, key: _ } => {
+            format!("Overridden by managed policy (MDM): {domain}")
+        }
+        ConfigLayerSource::System { file } => {
+            format!("Overridden by managed config (system): {}", file.display())
+        }
         ConfigLayerSource::SessionFlags => "Overridden by session flags".to_string(),
-        ConfigLayerSource::User { .. } => "Overridden by user config".to_string(),
+        ConfigLayerSource::User { file } => {
+            format!("Overridden by user config: {}", file.display())
+        }
+        ConfigLayerSource::LegacyManagedConfigTomlFromFile { file } => {
+            format!(
+                "Overridden by legacy managed_config.toml: {}",
+                file.display()
+            )
+        }
+        ConfigLayerSource::LegacyManagedConfigTomlFromMdm => {
+            "Overridden by legacy managed configuration from MDM".to_string()
+        }
     }
 }
 
@@ -513,7 +572,10 @@ fn compute_override_metadata(
     effective: &TomlValue,
     segments: &[String],
 ) -> Option<OverriddenMetadata> {
-    let user_value = value_at_path(&layers.user.config, segments);
+    let user_value = match layers.get_user_layer() {
+        Some(user_layer) => value_at_path(&user_layer.config, segments),
+        None => return None,
+    };
     let effective_value = value_at_path(effective, segments);
 
     if user_value.is_some() && user_value == effective_value {
@@ -524,8 +586,7 @@ fn compute_override_metadata(
         return None;
     }
 
-    let effective_layer = find_effective_layer(layers, segments);
-    let overriding_layer = effective_layer.unwrap_or_else(|| layers.user.metadata());
+    let overriding_layer = find_effective_layer(layers, segments)?;
     let message = override_message(&overriding_layer.name);
 
     Some(OverriddenMetadata {
@@ -554,23 +615,13 @@ fn find_effective_layer(
     layers: &ConfigLayerStack,
     segments: &[String],
 ) -> Option<ConfigLayerMetadata> {
-    let check =
-        |state: &ConfigLayerEntry| value_at_path(&state.config, segments).map(|_| state.metadata());
+    for layer in layers.layers_high_to_low() {
+        if let Some(meta) = value_at_path(&layer.config, segments).map(|_| layer.metadata()) {
+            return Some(meta);
+        }
+    }
 
-    if let Some(mdm) = &layers.mdm
-        && let Some(meta) = check(mdm)
-    {
-        return Some(meta);
-    }
-    if let Some(system) = &layers.system
-        && let Some(meta) = check(system)
-    {
-        return Some(meta);
-    }
-    if let Some(meta) = check(&layers.session_flags) {
-        return Some(meta);
-    }
-    check(&layers.user)
+    None
 }
 
 #[cfg(test)]
@@ -713,18 +764,18 @@ remote_compaction = true
                 .get("approval_policy")
                 .expect("origin")
                 .name,
-            ConfigLayerSource::System {
-                file: managed_file.clone(),
-            }
+            ConfigLayerSource::LegacyManagedConfigTomlFromFile {
+                file: managed_file.clone()
+            },
         );
         let layers = response.layers.expect("layers present");
+        assert_eq!(layers.len(), 2, "expected two layers");
         assert_eq!(
             layers.first().unwrap().name,
-            ConfigLayerSource::System { file: managed_file }
+            ConfigLayerSource::LegacyManagedConfigTomlFromFile { file: managed_file }
         );
-        assert_eq!(layers.get(1).unwrap().name, ConfigLayerSource::SessionFlags);
         assert_eq!(
-            layers.last().unwrap().name,
+            layers.get(1).unwrap().name,
             ConfigLayerSource::User { file: user_file }
         );
     }
@@ -779,7 +830,9 @@ remote_compaction = true
                 .get("approval_policy")
                 .expect("origin")
                 .name,
-            ConfigLayerSource::System { file: managed_file }
+            ConfigLayerSource::LegacyManagedConfigTomlFromFile {
+                file: managed_file.clone()
+            }
         );
         assert_eq!(result.status, WriteStatus::Ok);
         assert!(result.overridden_metadata.is_none());
@@ -909,14 +962,14 @@ remote_compaction = true
         assert_eq!(response.config.model.as_deref(), Some("system"));
         assert_eq!(
             response.origins.get("model").expect("origin").name,
-            ConfigLayerSource::System {
-                file: managed_file.clone(),
-            }
+            ConfigLayerSource::LegacyManagedConfigTomlFromFile {
+                file: managed_file.clone()
+            },
         );
         let layers = response.layers.expect("layers");
         assert_eq!(
             layers.first().unwrap().name,
-            ConfigLayerSource::System { file: managed_file }
+            ConfigLayerSource::LegacyManagedConfigTomlFromFile { file: managed_file }
         );
         assert_eq!(layers.get(1).unwrap().name, ConfigLayerSource::SessionFlags);
         assert_eq!(
@@ -959,7 +1012,7 @@ remote_compaction = true
         let overridden = result.overridden_metadata.expect("overridden metadata");
         assert_eq!(
             overridden.overriding_layer.name,
-            ConfigLayerSource::System { file: managed_file }
+            ConfigLayerSource::LegacyManagedConfigTomlFromFile { file: managed_file }
         );
         assert_eq!(overridden.effective_value, serde_json::json!("never"));
     }