From 94937de51ba28d4b308dbe1b8472d6fe1dddad28 Mon Sep 17 00:00:00 2001 From: charlesgong-openai Date: Sun, 9 Aug 2026 00:58:30 +0000 Subject: [PATCH] Improve plugin install failure analytics (#37645) ## Why Plugin install failures need stable, low-cardinality details that distinguish actionable causes without relying on error messages. ## What changed - Add HTTP status subtypes for remote catalog, mutation, and bundle download failures, including specific common statuses and a general `http_5xx` bucket. - Emit failure analytics when a remote plugin is disabled by an administrator or unavailable under its install policy. - Preserve marketplace I/O, configuration, and task failure subtypes, including failures that occur before a plugin ID can be resolved. ## Testing Add app-server coverage for catalog and install HTTP failures, unavailable remote plugins, unreadable marketplace files, and oversized bundle error responses. GitOrigin-RevId: 71a4888c492d1806bf4ff92bebc48f40cfb89300 --- .../src/request_processors/plugins.rs | 33 +++- .../tests/suite/v2/plugin_install.rs | 157 ++++++++++++++++++ codex-rs/core-plugins/src/error_subtype.rs | 13 ++ codex-rs/core-plugins/src/lib.rs | 1 + codex-rs/core-plugins/src/manager.rs | 46 ++++- codex-rs/core-plugins/src/remote.rs | 31 ++++ codex-rs/core-plugins/src/remote_bundle.rs | 5 +- codex-rs/core-plugins/src/remote_legacy.rs | 20 +++ 8 files changed, 294 insertions(+), 12 deletions(-) create mode 100644 codex-rs/core-plugins/src/error_subtype.rs diff --git a/codex-rs/app-server/src/request_processors/plugins.rs b/codex-rs/app-server/src/request_processors/plugins.rs index 45ff3c947a..d0f4f82458 100644 --- a/codex-rs/app-server/src/request_processors/plugins.rs +++ b/codex-rs/app-server/src/request_processors/plugins.rs @@ -1601,12 +1601,13 @@ impl PluginRequestProcessor { .await .map_err(|err| { let error_type = remote_plugin_catalog_error_type(&err); + let sub_error_type = err.sub_error_type(); self.track_plugin_install_failed_for_remote_plugin( &remote_plugin_id, &remote_marketplace_name, /*plugin_id*/ None, error_type, - /*sub_error_type*/ None, + sub_error_type, err.to_string(), ); remote_plugin_catalog_error_to_jsonrpc( @@ -1623,14 +1624,29 @@ impl PluginRequestProcessor { )) })?; if remote_detail.summary.availability == PluginAvailability::DisabledByAdmin { - return Err(invalid_request(format!( - "remote plugin {remote_plugin_id} is disabled by admin" - ))); + let error_message = format!("remote plugin {remote_plugin_id} is disabled by admin"); + self.track_plugin_install_failed_for_remote_plugin( + &remote_plugin_id, + &actual_remote_marketplace_name, + Some(&resolved_plugin_id), + "remote_plugin_not_available", + Some("disabled_by_admin".to_string()), + error_message.clone(), + ); + return Err(invalid_request(error_message)); } if remote_detail.summary.install_policy == PluginInstallPolicy::NotAvailable { - return Err(invalid_request(format!( - "remote plugin {remote_plugin_id} is not available for install" - ))); + let error_message = + format!("remote plugin {remote_plugin_id} is not available for install"); + self.track_plugin_install_failed_for_remote_plugin( + &remote_plugin_id, + &actual_remote_marketplace_name, + Some(&resolved_plugin_id), + "remote_plugin_not_available", + Some("install_policy_not_available".to_string()), + error_message.clone(), + ); + return Err(invalid_request(error_message)); } // Direct install writes the same cache tree that installed-plugin sync // prunes before the backend installed snapshot can include this plugin. @@ -1694,12 +1710,13 @@ impl PluginRequestProcessor { .await .map_err(|err| { let error_type = remote_plugin_catalog_error_type(&err); + let sub_error_type = err.sub_error_type(); self.track_plugin_install_failed_for_remote_plugin( &remote_plugin_id, &actual_remote_marketplace_name, Some(&result.plugin_id), error_type, - /*sub_error_type*/ None, + sub_error_type, err.to_string(), ); remote_plugin_catalog_error_to_jsonrpc(err, "install remote plugin") diff --git a/codex-rs/app-server/tests/suite/v2/plugin_install.rs b/codex-rs/app-server/tests/suite/v2/plugin_install.rs index 974d7de49b..38b85ee9cb 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_install.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_install.rs @@ -589,6 +589,79 @@ async fn plugin_install_tracks_analytics_when_remote_detail_fetch_fails() -> Res event_params["error_type"], "remote_catalog_unexpected_status" ); + assert_eq!(event_params["sub_error_type"], "http_404"); + Ok(()) +} + +#[tokio::test] +async fn plugin_install_tracks_analytics_when_remote_install_is_rate_limited() -> Result<()> { + let codex_home = TempDir::new()?; + let server = MockServer::start().await; + let bundle_url = mount_remote_plugin_bundle( + &server, + /*status_code*/ 200, + remote_plugin_bundle_tar_gz_bytes("linear")?, + ) + .await; + configure_remote_plugin_test(codex_home.path(), &server)?; + mount_remote_plugin_detail(&server, REMOTE_PLUGIN_ID, "1.2.3", Some(&bundle_url)).await; + mount_empty_remote_installed_plugins(&server).await; + Mock::given(method("POST")) + .and(path(format!( + "/backend-api/ps/plugins/{REMOTE_PLUGIN_ID}/install" + ))) + .and(header("authorization", "Bearer chatgpt-token")) + .and(header("chatgpt-account-id", "account-123")) + .respond_with(ResponseTemplate::new(/*status_code*/ 429).set_body_string("rate limited")) + .mount(&server) + .await; + mount_backend_analytics_events(&server).await; + + let mut mcp = TestAppServer::builder() + .with_codex_home(codex_home.path()) + .without_auto_env() + .with_env_overrides(&[(TEST_ALLOW_HTTP_REMOTE_PLUGIN_BUNDLE_DOWNLOADS, Some("1"))]) + .build_initialized_with_timeout(DEFAULT_TIMEOUT) + .await?; + + let request_id = send_remote_plugin_install_request(&mut mcp, REMOTE_PLUGIN_ID).await?; + let err = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(request_id)), + ) + .await??; + + assert_eq!(err.error.code, -32603); + assert!(err.error.message.contains("failed with status 429")); + wait_for_remote_plugin_request_count( + &server, + "GET", + "/bundles/linear.tar.gz", + /*expected_count*/ 1, + ) + .await?; + wait_for_remote_plugin_request_count( + &server, + "POST", + &format!("/ps/plugins/{REMOTE_PLUGIN_ID}/install"), + /*expected_count*/ 1, + ) + .await?; + let payload = wait_for_plugin_analytics_payload(&server).await?; + let event_params = &payload["events"][0]["event_params"]; + assert_eq!( + payload["events"][0]["event_type"], + "codex_plugin_install_failed" + ); + assert_eq!(event_params["plugin_id"], "linear@openai-curated-remote"); + assert_eq!(event_params["remote_plugin_id"], REMOTE_PLUGIN_ID); + assert_eq!(event_params["marketplace_name"], "openai-curated-remote"); + assert_eq!(event_params["source"], "manual"); + assert_eq!( + event_params["error_type"], + "remote_catalog_unexpected_status" + ); + assert_eq!(event_params["sub_error_type"], "http_429"); Ok(()) } @@ -612,6 +685,7 @@ async fn plugin_install_rejects_remote_plugin_disabled_by_admin_before_download( ) .await; mount_empty_remote_installed_plugins(&server).await; + mount_backend_analytics_events(&server).await; let mut mcp = TestAppServer::builder() .with_codex_home(codex_home.path()) @@ -649,6 +723,16 @@ async fn plugin_install_rejects_remote_plugin_disabled_by_admin_before_download( .join("plugins/cache/openai-curated-remote/linear") .exists() ); + let payload = wait_for_plugin_analytics_payload(&server).await?; + let event_params = &payload["events"][0]["event_params"]; + assert_eq!( + payload["events"][0]["event_type"], + "codex_plugin_install_failed" + ); + assert_eq!(event_params["plugin_id"], "linear@openai-curated-remote"); + assert_eq!(event_params["remote_plugin_id"], REMOTE_PLUGIN_ID); + assert_eq!(event_params["error_type"], "remote_plugin_not_available"); + assert_eq!(event_params["sub_error_type"], "disabled_by_admin"); Ok(()) } @@ -665,6 +749,7 @@ async fn plugin_install_rejects_remote_plugin_not_available() -> Result<()> { ) .await; mount_empty_remote_installed_plugins(&server).await; + mount_backend_analytics_events(&server).await; let mut mcp = TestAppServer::builder() .with_codex_home(codex_home.path()) @@ -688,6 +773,19 @@ async fn plugin_install_rejects_remote_plugin_not_available() -> Result<()> { /*expected_count*/ 0, ) .await?; + let payload = wait_for_plugin_analytics_payload(&server).await?; + let event_params = &payload["events"][0]["event_params"]; + assert_eq!( + payload["events"][0]["event_type"], + "codex_plugin_install_failed" + ); + assert_eq!(event_params["plugin_id"], "linear@openai-curated-remote"); + assert_eq!(event_params["remote_plugin_id"], REMOTE_PLUGIN_ID); + assert_eq!(event_params["error_type"], "remote_plugin_not_available"); + assert_eq!( + event_params["sub_error_type"], + "install_policy_not_available" + ); Ok(()) } @@ -792,6 +890,64 @@ async fn plugin_install_returns_invalid_request_for_missing_marketplace_file() - Ok(()) } +#[tokio::test] +async fn plugin_install_tracks_analytics_when_marketplace_file_cannot_be_read() -> Result<()> { + let analytics_server = start_analytics_events_server().await?; + let codex_home = TempDir::new()?; + write_analytics_config(codex_home.path(), &analytics_server.uri())?; + write_chatgpt_auth( + codex_home.path(), + ChatGptAuthFixture::new("chatgpt-token") + .account_id("account-123") + .chatgpt_user_id("user-123") + .chatgpt_account_id("account-123"), + AuthCredentialsStoreMode::File, + )?; + let marketplace_path = codex_home.path().join("marketplace-dir"); + std::fs::create_dir_all(&marketplace_path)?; + + let mut mcp = TestAppServer::builder() + .with_codex_home(codex_home.path()) + .without_auto_env() + .build_initialized_with_timeout(DEFAULT_TIMEOUT) + .await?; + + let request_id = mcp + .send_plugin_install_request(PluginInstallParams { + marketplace_path: Some(AbsolutePathBuf::try_from(marketplace_path)?), + remote_marketplace_name: None, + plugin_name: "sample-plugin".to_string(), + }) + .await?; + let err = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(request_id)), + ) + .await??; + + assert_eq!(err.error.code, -32603); + assert!( + err.error + .message + .contains("failed to read marketplace file") + ); + + let payload = wait_for_plugin_analytics_payload(&analytics_server).await?; + let event_params = &payload["events"][0]["event_params"]; + assert_eq!( + payload["events"][0]["event_type"], + "codex_plugin_install_failed" + ); + assert_eq!(event_params["plugin_id"], json!(null)); + assert_eq!(event_params["remote_plugin_id"], json!(null)); + assert_eq!(event_params["error_type"], "marketplace_io"); + assert_eq!( + event_params["sub_error_type"], + "failed_to_read_marketplace_file" + ); + Ok(()) +} + #[tokio::test] async fn plugin_install_returns_invalid_request_for_not_available_plugin() -> Result<()> { let codex_home = TempDir::new()?; @@ -1135,6 +1291,7 @@ async fn plugin_install_preserves_status_when_remote_bundle_error_body_is_too_la assert_eq!(event_params["marketplace_name"], "openai-curated-remote"); assert_eq!(event_params["source"], "manual"); assert_eq!(event_params["error_type"], "remote_bundle_download_status"); + assert_eq!(event_params["sub_error_type"], "http_5xx"); assert!( !codex_home .path() diff --git a/codex-rs/core-plugins/src/error_subtype.rs b/codex-rs/core-plugins/src/error_subtype.rs new file mode 100644 index 0000000000..130964f2d7 --- /dev/null +++ b/codex-rs/core-plugins/src/error_subtype.rs @@ -0,0 +1,13 @@ +use http::StatusCode; + +pub(crate) fn http_status_sub_error_type(status: StatusCode) -> &'static str { + match status.as_u16() { + 401 => "http_401", + 403 => "http_403", + 404 => "http_404", + 409 => "http_409", + 429 => "http_429", + _ if status.is_server_error() => "http_5xx", + _ => "http_other", + } +} diff --git a/codex-rs/core-plugins/src/lib.rs b/codex-rs/core-plugins/src/lib.rs index a31a0f6619..71d2ce70bb 100644 --- a/codex-rs/core-plugins/src/lib.rs +++ b/codex-rs/core-plugins/src/lib.rs @@ -1,6 +1,7 @@ mod app_mcp_routing; mod command_migration; mod discoverable; +mod error_subtype; mod http_client_selector; pub mod installed_marketplaces; pub mod loader; diff --git a/codex-rs/core-plugins/src/manager.rs b/codex-rs/core-plugins/src/manager.rs index 1519f6a0c2..48cb599929 100644 --- a/codex-rs/core-plugins/src/manager.rs +++ b/codex-rs/core-plugins/src/manager.rs @@ -66,6 +66,7 @@ use crate::startup_sync::sync_openai_plugins_repo; use crate::store::PluginInstallResult as StorePluginInstallResult; use crate::store::PluginStore; use crate::store::PluginStoreError; +use crate::store::error_context_sub_error_type; use crate::tool_suggest_metadata::ToolSuggestMetadataCache; use codex_analytics::AnalyticsEventsClient; use codex_analytics::PluginInstallSource; @@ -1555,6 +1556,7 @@ impl PluginsManager { } fn track_plugin_install_resolution_failed(&self, err: &MarketplaceError) { + let sub_error_type = marketplace_error_sub_error_type(err); let plugin_id = match err { MarketplaceError::PluginNotFound { plugin_name, @@ -1574,15 +1576,25 @@ impl PluginsManager { self.track_plugin_install_failed( &plugin_id, marketplace_error_type(err), - /*sub_error_type*/ None, + sub_error_type, err.to_string(), ); } else { tracing::warn!( error_type = %marketplace_error_type(err), + sub_error_type = sub_error_type.as_deref(), error = %err, "plugin install failed while resolving marketplace plugin" ); + self.emit_plugin_install_failed( + PluginTelemetryMetadata { + plugin_id: None, + remote_plugin_id: None, + capability_summary: None, + }, + marketplace_error_type(err), + sub_error_type, + ); } } @@ -1600,13 +1612,26 @@ impl PluginsManager { error = %error_message, "plugin install failed" ); + self.emit_plugin_install_failed( + self.telemetry_metadata_for_plugin_id(plugin_id), + error_type, + sub_error_type, + ); + } + + fn emit_plugin_install_failed( + &self, + plugin: PluginTelemetryMetadata, + error_type: &'static str, + sub_error_type: Option, + ) { let analytics_events_client = match self.analytics_events_client.read() { Ok(client) => client.clone(), Err(err) => err.into_inner().clone(), }; if let Some(analytics_events_client) = analytics_events_client { analytics_events_client.track_plugin_install_failed( - self.telemetry_metadata_for_plugin_id(plugin_id), + plugin, self.plugin_install_source, error_type.to_string(), sub_error_type, @@ -3053,8 +3078,11 @@ impl PluginInstallError { pub fn sub_error_type(&self) -> Option { match self { + Self::Marketplace(err) => marketplace_error_sub_error_type(err), + Self::Remote(err) => err.sub_error_type(), Self::Store(err) => err.sub_error_type(), - Self::Marketplace(_) | Self::Remote(_) | Self::Config(_) | Self::Join(_) => None, + Self::Config(_) => Some("failed_to_enable_plugin".to_string()), + Self::Join(_) => Some("plugin_install_task_failed".to_string()), } } } @@ -3081,6 +3109,18 @@ fn marketplace_error_type(err: &MarketplaceError) -> &'static str { } } +fn marketplace_error_sub_error_type(err: &MarketplaceError) -> Option { + match err { + MarketplaceError::Io { context, .. } => Some(error_context_sub_error_type(context)), + MarketplaceError::MarketplaceNotFound { .. } + | MarketplaceError::InvalidMarketplaceFile { .. } + | MarketplaceError::PluginNotFound { .. } + | MarketplaceError::PluginNotAvailable { .. } + | MarketplaceError::PluginsDisabled + | MarketplaceError::InvalidPlugin(_) => None, + } +} + fn remote_plugin_mutation_error_type(err: &RemotePluginMutationError) -> &'static str { match err { RemotePluginMutationError::AuthRequired => "remote_mutation_auth_required", diff --git a/codex-rs/core-plugins/src/remote.rs b/codex-rs/core-plugins/src/remote.rs index 457de514b2..f55fcf5f86 100644 --- a/codex-rs/core-plugins/src/remote.rs +++ b/codex-rs/core-plugins/src/remote.rs @@ -1,4 +1,5 @@ use crate::app_mcp_routing::apply_app_mcp_routing_policy; +use crate::error_subtype::http_status_sub_error_type; use crate::http_client_selector::HttpClientSelector; use crate::loader::plugin_app_declarations_from_value; use crate::store::PLUGINS_CACHE_DIR; @@ -458,6 +459,36 @@ pub enum RemotePluginCatalogError { CacheRemove(String), } +impl RemotePluginCatalogError { + /// Stable low-cardinality detail for plugin-install failure telemetry. + pub fn sub_error_type(&self) -> Option { + match self { + Self::UnexpectedStatus { status, .. } => { + Some(http_status_sub_error_type(*status).to_string()) + } + Self::AuthRequired + | Self::UnsupportedAuthMode + | Self::AuthToken(_) + | Self::Request { .. } + | Self::Decode { .. } + | Self::InvalidBaseUrl(_) + | Self::InvalidBaseUrlPath + | Self::UnknownMarketplace { .. } + | Self::UnexpectedPluginId { .. } + | Self::UnexpectedSkillName { .. } + | Self::UnexpectedEnabledState { .. } + | Self::InvalidPluginPath { .. } + | Self::PluginShareCheckoutNotAvailable { .. } + | Self::Archive { .. } + | Self::ArchiveJoin(_) + | Self::ArchiveTooLarge { .. } + | Self::MissingUploadEtag + | Self::UnexpectedResponse(_) + | Self::CacheRemove(_) => None, + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)] pub enum RemotePluginScope { #[serde(rename = "GLOBAL")] diff --git a/codex-rs/core-plugins/src/remote_bundle.rs b/codex-rs/core-plugins/src/remote_bundle.rs index 25e136f8a1..8a3c1f47c6 100644 --- a/codex-rs/core-plugins/src/remote_bundle.rs +++ b/codex-rs/core-plugins/src/remote_bundle.rs @@ -1,3 +1,4 @@ +use crate::error_subtype::http_status_sub_error_type; use crate::plugin_bundle_archive::PluginBundleUnpackError; use crate::plugin_bundle_archive::unpack_plugin_bundle_tar_gz; use crate::remote::REMOTE_GLOBAL_MARKETPLACE_NAME; @@ -143,6 +144,9 @@ impl RemotePluginBundleInstallError { match self { Self::Io { context, .. } => Some(error_context_sub_error_type(context)), Self::Store(err) => err.sub_error_type(), + Self::DownloadStatus { status, .. } => { + Some(http_status_sub_error_type(*status).to_string()) + } Self::MissingReleaseVersion { .. } | Self::InvalidReleaseVersion { .. } | Self::MissingBundleDownloadUrl { .. } @@ -150,7 +154,6 @@ impl RemotePluginBundleInstallError { | Self::UnsupportedBundleDownloadUrlScheme { .. } | Self::InvalidPluginId { .. } | Self::DownloadRequest { .. } - | Self::DownloadStatus { .. } | Self::DownloadBody { .. } | Self::DownloadTooLarge { .. } | Self::UnsupportedBundleDownloadFinalUrl { .. } diff --git a/codex-rs/core-plugins/src/remote_legacy.rs b/codex-rs/core-plugins/src/remote_legacy.rs index 5701637aae..65f1c04efd 100644 --- a/codex-rs/core-plugins/src/remote_legacy.rs +++ b/codex-rs/core-plugins/src/remote_legacy.rs @@ -1,3 +1,4 @@ +use crate::error_subtype::http_status_sub_error_type; use crate::remote::RemotePluginServiceConfig; use codex_http_client::RouteAwareRequestError; use codex_login::CodexAuth; @@ -73,6 +74,25 @@ pub enum RemotePluginMutationError { }, } +impl RemotePluginMutationError { + pub(crate) fn sub_error_type(&self) -> Option { + match self { + Self::UnexpectedStatus { status, .. } => { + Some(http_status_sub_error_type(*status).to_string()) + } + Self::AuthRequired + | Self::UnsupportedAuthMode + | Self::AuthToken(_) + | Self::InvalidBaseUrl(_) + | Self::InvalidBaseUrlPath + | Self::Request { .. } + | Self::Decode { .. } + | Self::UnexpectedPluginId { .. } + | Self::UnexpectedEnabledState { .. } => None, + } + } +} + #[derive(Debug, thiserror::Error)] pub enum RemotePluginFetchError { #[error("invalid chatgpt base url for remote featured plugin request: {0}")]