Preserve UTF-8 byte semantics in network domain matching (#52277)

## Why

Domain wildcards need to retain the byte-based behavior of `globset`. Counting Unicode scalar values changes how `?` matches non-ASCII hosts.

## What changed

Match domain patterns against UTF-8 bytes so `?` consumes exactly one byte. For example, `api??.com` matches `apié.com`. Match hosts through borrowed byte slices, avoiding the per-match character-vector allocation.

Document ASCII case insensitivity and the absence of IDNA conversion: hosts supplied as Punycode require patterns using their Punycode spelling.

## Testing

Compare Unicode wildcard cases against `globset` for both allowlists and denylists, including multibyte characters, combining marks, Punycode, and case sensitivity. Update the exhaustive dynamic-programming comparison and retry-boundary tests to operate on bytes.

GitOrigin-RevId: fae5b3b1ae60b469cd61a44c5c6e245730adc8ba

codex-rs/network-proxy/README.md

@@ -241,9 +241,12 @@ what it can reasonably guarantee.
 
 - Allowlist-first policy: if `domains` has no `allow` entries, requests are blocked until an allowlist is configured.
 - Domain patterns: exact hosts are supported, `*.example.com` matches subdomains only, and `**.example.com` matches the apex plus subdomains; the global `*` wildcard is only accepted when explicitly enabled for allowlist compilation and is otherwise rejected.
-- Within a domain pattern, `?` matches exactly one character, including a dot. For example,
+- Within a domain pattern, `?` matches exactly one byte, including a dot. For example,
   `api?.example.com` matches `api1.example.com`, but not `api.example.com` or `api12.example.com`.
   It can be combined with `*`, `*.`, and `**.` in both allow and deny entries.
+  Matching uses UTF-8 bytes with ASCII case insensitivity; it does not convert between Unicode
+  domain names and IDNA/Punycode. Internationalized domains arriving as Punycode must be matched
+  using their Punycode spelling.
 - Deny wins: `domains` entries marked `deny` always override the allowlist.
 - Local/private network protection: when `allow_local_binding = false`, the proxy blocks loopback
   and common private/link-local ranges. Explicit allowlisting of local IP literals (or `localhost`)

codex-rs/network-proxy/src/domain_matcher.rs

@@ -1,6 +1,7 @@
-//! Network-domain wildcards over Unicode scalar values, independent of filesystem semantics.
+//! Network-domain wildcards over UTF-8 bytes, independent of filesystem semantics.
 //! Construction validates and normalizes patterns and expands domain prefixes.
-//! Callers normalize candidate hosts before matching.
+//! Callers normalize candidate hosts before matching. No IDNA conversion is performed.
+//! Byte matching preserves the historical globset wildcard semantics.
 
 use anyhow::Result;
 use anyhow::bail;
@@ -18,7 +19,7 @@ pub(crate) enum GlobalWildcard {
 
 #[derive(Clone)]
 pub struct DomainPatternSet {
-    patterns: Arc<[Vec<char>]>,
+    patterns: Arc<[Vec<u8>]>,
 }
 
 impl DomainPatternSet {
@@ -65,14 +66,14 @@ impl DomainPatternSet {
             // - "example.com": match the exact host
             // - "*.example.com": match any subdomain (not the apex)
             // - "**.example.com": match the apex and any subdomain
-            // - "api?.example.com": match exactly one Unicode scalar value after "api"
+            // - "api?.example.com": match exactly one byte after "api"
             // - "*": match every host when explicitly enabled for allowlist compilation
             for candidate in expand_domain_pattern(&pattern) {
                 if !seen.insert(candidate.clone()) {
                     continue;
                 }
                 ensure!(!candidate.is_empty(), "domain pattern is empty");
-                compiled.push(candidate.chars().collect());
+                compiled.push(candidate.into_bytes());
             }
         }
         Ok(Self {
@@ -81,8 +82,8 @@ impl DomainPatternSet {
     }
 
     pub fn is_match(&self, host: impl AsRef<str>) -> bool {
-        let host: Vec<char> = host.as_ref().chars().collect();
-        self.patterns.iter().any(|pattern| matches(pattern, &host))
+        let host = host.as_ref().as_bytes();
+        self.patterns.iter().any(|pattern| matches(pattern, host))
     }
 }
 
@@ -186,16 +187,16 @@ fn expand_domain_pattern(pattern: &str) -> Vec<String> {
 
 // Only the latest star needs retrying: it can absorb any extension that an earlier star could.
 // Its retry position advances monotonically, bounding work by O(pattern length * host length).
-fn matches(pattern: &[char], host: &[char]) -> bool {
+fn matches(pattern: &[u8], host: &[u8]) -> bool {
     let (mut p, mut h) = (0, 0);
     let mut star = None;
     while h < host.len() {
         match pattern.get(p) {
-            Some('*') => {
+            Some(b'*') => {
                 star = Some((p + 1, h));
                 p += 1;
             }
-            Some(c) if *c == '?' || c.eq_ignore_ascii_case(&host[h]) => {
+            Some(c) if *c == b'?' || c.eq_ignore_ascii_case(&host[h]) => {
                 p += 1;
                 h += 1;
             }
@@ -209,7 +210,7 @@ fn matches(pattern: &[char], host: &[char]) -> bool {
             },
         }
     }
-    pattern[p..].iter().all(|c| *c == '*')
+    pattern[p..].iter().all(|c| *c == b'*')
 }
 
 #[cfg(test)]

codex-rs/network-proxy/src/domain_matcher_tests.rs

@@ -5,19 +5,36 @@ use crate::policy::normalize_host;
 use pretty_assertions::assert_eq;
 
 #[test]
-fn unicode_wildcards_count_scalars_not_bytes_or_graphemes() {
+fn unicode_wildcards_preserve_utf8_byte_matching() {
     for (pattern, host, expected) in [
-        ("api?.com", "apié.com", true),
-        ("api?.com", "api😀.com", true),
-        ("api??.com", "apié.com", false),
-        ("api?.com", "apie\u{301}.com", false),
-        ("api??.com", "apie\u{301}.com", true),
-        ("*.例?.com", "a.例子.com", true),
-        ("*.例?.com", "例子.com", false),
-        ("**.例?.com", "例子.com", true),
+        ("api?.com", "api1.com", true),
+        ("api??.com", "apié.com", true),
+        ("api????.com", "api😀.com", true),
+        ("api???.com", "apie\u{301}.com", true),
+        ("api?*.com", "apié.com", true),
+        ("*.例???.com", "a.例子.com", true),
+        ("*.例???.com", "例子.com", false),
+        ("**.例???.com", "例子.com", true),
+        ("büch?.de", "büch1.de", true),
+        ("bücher.de", "bücher.de", true),
+        ("bücher.de", "xn--bcher-kva.de", false),
+        ("xn--*.de", "xn--bcher-kva.de", true),
         ("É.com", "é.com", false),
         ("É.COM", "É.com", true),
     ] {
+        let mut legacy = globset::GlobSetBuilder::new();
+        for expanded in expand_domain_pattern(&normalize_pattern(pattern)) {
+            legacy.add(
+                globset::GlobBuilder::new(&expanded)
+                    .case_insensitive(true)
+                    .build()
+                    .unwrap(),
+            );
+        }
+        assert_eq!(
+            legacy.build().unwrap().is_match(normalize_host(host)),
+            expected
+        );
         for compile in [compile_allowlist, compile_denylist] {
             assert_eq!(
                 compile(&[pattern.to_string()])
@@ -105,7 +122,7 @@ fn bracketed_patterns_preserve_port_and_trailing_dot_normalization() {
 
 #[test]
 fn matcher_agrees_with_independent_dynamic_program_for_short_inputs() {
-    fn words(alphabet: &[char], max_len: usize) -> Vec<Vec<char>> {
+    fn words(alphabet: &[u8], max_len: usize) -> Vec<Vec<u8>> {
         let mut all = vec![vec![]];
         let mut level = vec![vec![]];
         for _ in 0..max_len {
@@ -123,23 +140,23 @@ fn matcher_agrees_with_independent_dynamic_program_for_short_inputs() {
         }
         all
     }
-    fn reference(pattern: &[char], host: &[char]) -> bool {
+    fn reference(pattern: &[u8], host: &[u8]) -> bool {
         let mut rows = vec![vec![false; host.len() + 1]; pattern.len() + 1];
         rows[0][0] = true;
         for (p, c) in pattern.iter().enumerate() {
-            rows[p + 1][0] = *c == '*' && rows[p][0];
+            rows[p + 1][0] = *c == b'*' && rows[p][0];
             for (h, candidate) in host.iter().enumerate() {
-                rows[p + 1][h + 1] = if *c == '*' {
+                rows[p + 1][h + 1] = if *c == b'*' {
                     rows[p][h + 1] || rows[p + 1][h]
                 } else {
-                    rows[p][h] && (*c == '?' || c.eq_ignore_ascii_case(candidate))
+                    rows[p][h] && (*c == b'?' || c.eq_ignore_ascii_case(candidate))
                 };
             }
         }
         rows[pattern.len()][host.len()]
     }
-    for pattern in words(&['a', 'é', '?', '*'], /*max_len*/ 4) {
-        for host in words(&['a', 'é'], /*max_len*/ 5) {
+    for pattern in words(&[b'a', 0xc3, 0xa9, b'?', b'*'], /*max_len*/ 4) {
+        for host in words(&[b'a', 0xc3, 0xa9], /*max_len*/ 5) {
             assert_eq!(
                 matches(&pattern, &host),
                 reference(&pattern, &host),
@@ -151,17 +168,13 @@ fn matcher_agrees_with_independent_dynamic_program_for_short_inputs() {
 
 #[test]
 fn adversarial_retries_and_empty_boundaries() {
-    let pattern: Vec<char> = format!("*{}b", "a".repeat(1024)).chars().collect();
-    let host = vec!['a'; 4096];
+    let pattern = format!("*{}b", "a".repeat(1024)).into_bytes();
+    let host = vec![b'a'; 4096];
     assert!(!matches(&pattern, &host));
-    let pattern: Vec<char> = format!("{}b", "*a".repeat(1024)).chars().collect();
+    let pattern = format!("{}b", "*a".repeat(1024)).into_bytes();
     assert!(!matches(&pattern, &host));
     assert_eq!(
-        (
-            matches(&[], &[]),
-            matches(&['*'], &[]),
-            matches(&['?'], &[])
-        ),
+        (matches(&[], &[]), matches(b"*", &[]), matches(b"?", &[])),
         (true, true, false)
     );
     assert!(!compile_allowlist(&[]).unwrap().is_match("host"));