Record Windows sandbox service stop diagnostics (#50507)

## What changed

Persist the service's last reported lifecycle reason and numeric HRESULT in the existing service registry key. Distinguish requested stops, shutdown, owner removal, startup failure, broker failure, and interrupted registration, and reset the record on startup. Recording is best-effort and does not affect the service lifecycle.

Emit `codex.windows_sandbox.service_stopped` when a provisioning client observes a stopped service, including the recorded reason, HRESULT, and Windows service exit codes. Recheck the service status and registry record before emitting to avoid pairing inconsistent evidence during a concurrent restart. The metric counts observations, not unique outages; the recorded reason is advisory.

Update the cached service status only after `SetServiceStatus` succeeds. Extract diagnostic error codes from typed errors without collecting error text or user data.

## Testing

Add a test covering Win32-to-HRESULT conversion, preservation of Windows HRESULTs through error context, and a zero fallback for untyped error text.

GitOrigin-RevId: d0d39d091410e2b5499a0803475804d12235f836

codex-rs/windows-sandbox-rs/src/lib.rs

@@ -108,6 +108,8 @@ mod provisioning_protocol;
 #[cfg(target_os = "windows")]
 mod runtime_ownership;
 #[cfg(target_os = "windows")]
+mod service_diagnostics;
+#[cfg(target_os = "windows")]
 mod service_identity;
 #[cfg(target_os = "windows")]
 #[doc(hidden)]
@@ -134,6 +136,9 @@ pub use runtime_ownership::remove_installation;
 #[doc(hidden)]
 pub use runtime_ownership::save_installation;
 #[cfg(target_os = "windows")]
+#[doc(hidden)]
+pub use service_diagnostics::ServiceStopReason;
+#[cfg(target_os = "windows")]
 mod resolved_permissions;
 #[cfg(target_os = "windows")]
 mod token;

codex-rs/windows-sandbox-rs/src/provisioning_client.rs

@@ -419,6 +419,9 @@ fn query_service_status() -> anyhow::Result<Services::SERVICE_STATUS_PROCESS> {
     {
         return Err(io::Error::last_os_error()).context("query sandbox provisioning service");
     }
+    if status.dwCurrentState == Services::SERVICE_STOPPED {
+        crate::service_diagnostics::record_stopped(&status, service.0);
+    }
     Ok(status)
 }
 

codex-rs/windows-sandbox-rs/src/service_diagnostics.rs

@@ -0,0 +1,129 @@
+//! Best-effort, numeric service-stop evidence. Core exports it only when SCM says stopped.
+//! Missing/older records stay unknown; no event-log text or user data is collected.
+
+use std::mem::size_of_val;
+use std::ptr;
+use windows_sys::Win32::System::Registry as registry;
+use windows_sys::Win32::System::Services as services;
+use windows_sys::Win32::System::Services::SERVICE_STATUS_PROCESS;
+
+/// Last lifecycle outcome reported by the service, not an inferred crash diagnosis.
+#[derive(Clone, Copy)]
+#[repr(u32)]
+pub enum ServiceStopReason {
+    Starting = 1,
+    StopRequested = 2,
+    Shutdown = 3,
+    OwnerRemoved = 4,
+    StartupFailed = 5,
+    BrokerFailed = 6,
+    RegistrationInterrupted = 7,
+}
+
+impl ServiceStopReason {
+    /// Called by the service, which owns the key. Failure must not affect its lifecycle.
+    pub fn record(self, hresult: u32) {
+        let Ok(service_name) = crate::windows_sandbox_service_name() else {
+            return;
+        };
+        let key = crate::to_wide(format!(r"SYSTEM\CurrentControlSet\Services\{service_name}"));
+        let value = ((self as u64) << 32) | u64::from(hresult);
+        let mut handle = ptr::null_mut();
+        unsafe {
+            // Do not recreate a service key that Windows is uninstalling.
+            if registry::RegOpenKeyExW(
+                registry::HKEY_LOCAL_MACHINE,
+                key.as_ptr(),
+                0,
+                registry::KEY_SET_VALUE,
+                &mut handle,
+            ) != 0
+            {
+                return;
+            }
+            registry::RegSetValueExW(
+                handle,
+                windows_sys::w!("CodexLastStop"),
+                0,
+                registry::REG_QWORD,
+                ptr::from_ref(&value).cast(),
+                size_of_val(&value) as u32,
+            );
+            registry::RegCloseKey(handle);
+        }
+    }
+}
+
+pub(crate) fn record_stopped(status: &SERVICE_STATUS_PROCESS, service: services::SC_HANDLE) {
+    let Some(metrics) = codex_otel::global() else {
+        return;
+    };
+    let Ok(service_name) = crate::windows_sandbox_service_name() else {
+        return;
+    };
+    let key = crate::to_wide(format!(r"SYSTEM\CurrentControlSet\Services\{service_name}"));
+    let read_stop = || {
+        let mut value = 0_u64;
+        let mut size = size_of_val(&value) as u32;
+        unsafe {
+            registry::RegGetValueW(
+                registry::HKEY_LOCAL_MACHINE,
+                key.as_ptr(),
+                windows_sys::w!("CodexLastStop"),
+                registry::RRF_RT_REG_QWORD | registry::RRF_ZEROONFAILURE,
+                ptr::null_mut(),
+                ptr::from_mut(&mut value).cast(),
+                &mut size,
+            );
+        }
+        value
+    };
+    let value = read_stop();
+    // A concurrent restart can replace the registry record after the first query.
+    // Only pair it with SCM evidence if the stopped state and exit codes still match.
+    let mut current: SERVICE_STATUS_PROCESS = unsafe { std::mem::zeroed() };
+    let mut bytes_needed = 0;
+    if unsafe {
+        services::QueryServiceStatusEx(
+            service,
+            services::SC_STATUS_PROCESS_INFO,
+            ptr::from_mut(&mut current).cast(),
+            size_of_val(&current) as u32,
+            &mut bytes_needed,
+        )
+    } == 0
+        || current.dwCurrentState != services::SERVICE_STOPPED
+        || current.dwWin32ExitCode != status.dwWin32ExitCode
+        || current.dwServiceSpecificExitCode != status.dwServiceSpecificExitCode
+        || read_stop() != value
+    {
+        return;
+    }
+    // This counts observations, not unique outages. SCM's exit code also covers
+    // crashes/kills that cannot write a final record. The registry is advisory.
+    let _ = metrics.counter(
+        "codex.windows_sandbox.service_stopped",
+        /*inc*/ 1,
+        &[
+            (
+                "last_reported_reason",
+                match (value >> 32) as u32 {
+                    1 => "no_stop_recorded",
+                    2 => "stop_requested",
+                    3 => "shutdown",
+                    4 => "owner_removed",
+                    5 => "startup_failed",
+                    6 => "broker_failed",
+                    7 => "registration_interrupted",
+                    _ => "unknown",
+                },
+            ),
+            ("last_reported_hresult", &format!("0x{:08x}", value as u32)),
+            ("win32_exit_code", &status.dwWin32ExitCode.to_string()),
+            (
+                "service_exit_code",
+                &status.dwServiceSpecificExitCode.to_string(),
+            ),
+        ],
+    );
+}

codex-rs/windows-sandbox-service/src/service.rs

@@ -11,6 +11,7 @@ use std::sync::atomic::Ordering;
 
 use anyhow::Context;
 use anyhow::Result;
+use codex_windows_sandbox::ServiceStopReason;
 use windows_sys::Win32::Foundation::ERROR_CALL_NOT_IMPLEMENTED;
 use windows_sys::Win32::Foundation::ERROR_SERVICE_SPECIFIC_ERROR;
 use windows_sys::Win32::Foundation::NO_ERROR;
@@ -135,7 +136,16 @@ unsafe extern "system" fn service_main(_argument_count: u32, _arguments: *mut *m
         return;
     };
 
+    ServiceStopReason::Starting.record(/*hresult*/ 0);
     if let Err(error) = service_main_inner(state) {
+        let reason = if error.is::<crate::registered_runtime::RegistrationInterrupted>() {
+            ServiceStopReason::RegistrationInterrupted
+        } else if state.current_status.load(Ordering::Acquire) == SERVICE_START_PENDING {
+            ServiceStopReason::StartupFailed
+        } else {
+            ServiceStopReason::BrokerFailed
+        };
+        reason.record(fatal_error_hresult(&error));
         log_error(
             EVENT_SERVICE_FAILED,
             &format!("The Codex sandbox service encountered a fatal error: {error:#}"),
@@ -173,7 +183,8 @@ fn service_main_inner(state: &ServiceState) -> Result<()> {
     state.report_status(SERVICE_START_PENDING, NO_ERROR)?;
     let package_lifecycle =
         crate::package_lifecycle::PackageLifecycle::new(Arc::clone(&state.uninstalling))?;
-    runtime_lifecycle::run(state, &package_lifecycle)?;
+    let reason = runtime_lifecycle::run(state, &package_lifecycle)?;
+    reason.record(/*hresult*/ 0);
     log_information(
         EVENT_SERVICE_STOPPED,
         "The Codex sandbox service has stopped.",
@@ -181,6 +192,22 @@ fn service_main_inner(state: &ServiceState) -> Result<()> {
     state.report_status(SERVICE_STOPPED, NO_ERROR)
 }
 
+fn fatal_error_hresult(error: &anyhow::Error) -> u32 {
+    // Never stringify an error: contexts can contain paths, accounts or credentials.
+    error
+        .chain()
+        .find_map(|cause| {
+            if let Some(error) = cause.downcast_ref::<windows::core::Error>() {
+                return Some(error.code().0 as u32);
+            }
+            cause
+                .downcast_ref::<io::Error>()?
+                .raw_os_error()
+                .map(|code| windows::core::HRESULT::from_win32(code as u32).0 as u32)
+        })
+        .unwrap_or(0)
+}
+
 unsafe extern "system" fn service_control_handler(
     control: u32,
     _event_type: u32,
@@ -314,12 +341,12 @@ impl ServiceState {
             dwWaitHint: if is_pending { 10_000 } else { 0 },
         };
 
-        self.current_status.store(current_status, Ordering::Release);
         // The SCM synchronously copies the status structure during this call.
         let updated = unsafe { SetServiceStatus(status_handle.0, &status) };
         if updated == 0 {
             return Err(io::Error::last_os_error()).context("update the Windows service status");
         }
+        self.current_status.store(current_status, Ordering::Release);
 
         Ok(())
     }

codex-rs/windows-sandbox-service/src/service/runtime_lifecycle.rs

@@ -7,6 +7,7 @@ use std::sync::atomic::Ordering;
 
 use anyhow::Context;
 use anyhow::Result;
+use codex_windows_sandbox::ServiceStopReason;
 use windows_sys::Win32::Foundation::NO_ERROR;
 use windows_sys::Win32::System::Services::SERVICE_RUNNING;
 use windows_sys::Win32::System::Services::SERVICE_STOP_PENDING;
@@ -51,7 +52,10 @@ pub(super) fn foreground_owner(
     Ok(record)
 }
 
-pub(super) fn run(state: &ServiceState, package_lifecycle: &PackageLifecycle) -> Result<()> {
+pub(super) fn run(
+    state: &ServiceState,
+    package_lifecycle: &PackageLifecycle,
+) -> Result<ServiceStopReason> {
     let cleaned = Cell::new(false);
     let last_cleanup_error = Cell::new(None);
     let restore_owner = || -> Result<()> {
@@ -117,7 +121,13 @@ pub(super) fn run(state: &ServiceState, package_lifecycle: &PackageLifecycle) ->
     {
         package_lifecycle.clean_up()?;
     }
-    Ok(())
+    Ok(if cleaned.get() {
+        ServiceStopReason::OwnerRemoved
+    } else if state.stop_requested.load(Ordering::Acquire) {
+        ServiceStopReason::StopRequested
+    } else {
+        ServiceStopReason::Shutdown
+    })
 }
 
 /// Retries the current teardown step, never a phase recovered from stored intent.

codex-rs/windows-sandbox-service/src/service/runtime_lifecycle_tests.rs

@@ -3,6 +3,7 @@
 use super::retry_cleanup;
 use crate::service::SERVICE_STATE;
 use crate::service::ServiceState;
+use crate::service::fatal_error_hresult;
 use crate::service::service_control_handler;
 use anyhow::Context;
 use anyhow::Result;
@@ -20,6 +21,29 @@ use windows_sys::Win32::System::Services::SERVICE_CONTROL_SHUTDOWN;
 use windows_sys::Win32::System::Services::SERVICE_CONTROL_STOP;
 use windows_sys::Win32::System::Services::SERVICE_STOPPED;
 
+#[test]
+fn fatal_diagnostics_keep_typed_codes_without_error_text() {
+    let errors = [
+        (
+            anyhow::Error::new(io::Error::from_raw_os_error(5)),
+            0x80070005,
+        ),
+        (
+            anyhow::Error::new(windows::core::Error::from_hresult(windows::core::HRESULT(
+                0x80073cf3_u32 as i32,
+            ))),
+            0x80073cf3,
+        ),
+        (anyhow::anyhow!("token=private (os error 5)"), 0),
+    ];
+    for (error, expected) in errors {
+        assert_eq!(
+            fatal_error_hresult(&error.context("private path/account")),
+            expected
+        );
+    }
+}
+
 #[test]
 fn cleanup_retries_after_scm_stop_but_not_system_shutdown() -> Result<()> {
     // One test owns the process-global state. The pre-set shutdown flag keeps