From e2624fb65ca421e18d6fb8fc2986074733311b9a Mon Sep 17 00:00:00 2001 From: Brisbanehuang <202969697+Brisbanehuang@users.noreply.github.com> Date: Mon, 31 Aug 2026 09:42:58 -0400 Subject: [PATCH] fix(codex): preserve known image input capabilities --- .../openai_codex_models_handler_test.go | 4 +- .../service/openai_codex_models_service.go | 36 +++++-- .../openai_codex_models_service_test.go | 100 +++++++++++++++++- 3 files changed, 129 insertions(+), 11 deletions(-) diff --git a/backend/internal/handler/openai_codex_models_handler_test.go b/backend/internal/handler/openai_codex_models_handler_test.go index 9eabd3e5b2..686c6c13c5 100644 --- a/backend/internal/handler/openai_codex_models_handler_test.go +++ b/backend/internal/handler/openai_codex_models_handler_test.go @@ -620,8 +620,8 @@ func requireCompleteCodexModelsHandlerResponse(t *testing.T, recorder *httptest. t.Fatalf("truncation_policy must be populated: %v", model["truncation_policy"]) } modalities, ok := model["input_modalities"].([]any) - if !ok || len(modalities) != 1 || modalities[0] != "text" { - t.Fatalf("custom OpenAI-compatible endpoint modalities: got %v, want [text]", model["input_modalities"]) + if !ok || len(modalities) != 2 || modalities[0] != "text" || modalities[1] != "image" { + t.Fatalf("known GPT model modalities: got %v, want [text image]", model["input_modalities"]) } } diff --git a/backend/internal/service/openai_codex_models_service.go b/backend/internal/service/openai_codex_models_service.go index b55549e753..6acb7d242c 100644 --- a/backend/internal/service/openai_codex_models_service.go +++ b/backend/internal/service/openai_codex_models_service.go @@ -1041,6 +1041,11 @@ func accountCodexModelSupportsImageInput(account *Account, upstreamModel string) } switch account.Platform { case PlatformOpenAI: + if metadata, ok := account.GetUpstreamModelMetadata(upstreamModel); ok { + if modalities := normalizeCodexInputModalities(metadata.InputModalities); len(modalities) > 0 { + return stringSliceContains(modalities, "image") + } + } if !isOpenAICodexImageInputModel(upstreamModel) { return false } @@ -1050,8 +1055,9 @@ func accountCodexModelSupportsImageInput(account *Account, upstreamModel string) if !account.IsOpenAIApiKey() { return false } - baseURL := strings.TrimSpace(account.GetCredential("base_url")) - return baseURL == "" || isOfficialOpenAIModelsBaseURL(baseURL) + // Compatible model lists often omit modalities. Preserve the known GPT + // fallback unless a synced snapshot above explicitly narrows it. + return true case PlatformGrok: if !isOfficialGrokCodexBaseURL(account.GetGrokBaseURL()) { return false @@ -1715,7 +1721,7 @@ func (s *OpenAIGatewayService) fetchCodexModelsManifestUpstream(ctx context.Cont upstreamBody := body convertedFromOpenAIModelList := false if request.useAPIKeyUpstream { - convertedBody := convertOpenAIModelListToCodexManifest(body) + convertedBody := convertOpenAIModelListToCodexManifestForAccount(body, request.credentialAccount) convertedFromOpenAIModelList = !bytes.Equal(convertedBody, body) body = convertedBody } @@ -1734,7 +1740,7 @@ func (s *OpenAIGatewayService) fetchCodexModelsManifestUpstream(ctx context.Cont body, err = completeAPIKeyCodexModelsManifestMetadata( body, false, - request.credentialAccount != nil && isOfficialOpenAIModelsBaseURL(request.credentialAccount.GetOpenAIBaseURL()), + request.credentialAccount, ) if err != nil { return nil, &codexModelsManifestUpstreamError{ @@ -1855,6 +1861,10 @@ func adjustAPIKeyCodexModelsManifest(body []byte) ([]byte, error) { // standard list shape, or yield no usable model IDs are returned unchanged so // envelope validation reports the original payload. func convertOpenAIModelListToCodexManifest(body []byte) []byte { + return convertOpenAIModelListToCodexManifestForAccount(body, nil) +} + +func convertOpenAIModelListToCodexManifestForAccount(body []byte, account *Account) []byte { var envelope map[string]json.RawMessage if err := json.Unmarshal(body, &envelope); err != nil || envelope == nil { return body @@ -1883,7 +1893,13 @@ func convertOpenAIModelListToCodexManifest(body []byte) []byte { if len(modelIDs) == 0 { return body } - converted, err := BuildCodexModelsManifest(modelIDs) + imageInputModels := make(map[string]bool, len(modelIDs)) + for _, modelID := range modelIDs { + if accountCodexModelSupportsImageInput(account, modelID) { + imageInputModels[modelID] = true + } + } + converted, err := buildCodexModelsManifest(modelIDs, imageInputModels, nil, nil) if err != nil { return body } @@ -1904,7 +1920,7 @@ func (s *OpenAIGatewayService) CompleteAPIKeyCodexModelsManifestForClient(manife if len(manifest.upstreamSourceBody) > 0 { body = append([]byte(nil), manifest.upstreamSourceBody...) if manifest.convertedFromOpenAIModelList { - body = convertOpenAIModelListToCodexManifest(body) + body = convertOpenAIModelListToCodexManifestForAccount(body, account) } } var err error @@ -1915,7 +1931,7 @@ func (s *OpenAIGatewayService) CompleteAPIKeyCodexModelsManifestForClient(manife body, err = completeAPIKeyCodexModelsManifestMetadata( body, true, - isOfficialOpenAIModelsBaseURL(account.GetOpenAIBaseURL()), + account, ) if err != nil { return err @@ -2034,7 +2050,7 @@ func applySyncedAPIKeyCodexModelMetadata(body []byte, account *Account, overwrit return updated, nil } -func completeAPIKeyCodexModelsManifestMetadata(body []byte, completeAll, officialOpenAI bool) ([]byte, error) { +func completeAPIKeyCodexModelsManifestMetadata(body []byte, completeAll bool, account *Account) ([]byte, error) { var envelope map[string]json.RawMessage if err := json.Unmarshal(body, &envelope); err != nil { return nil, fmt.Errorf("decode JSON object: %w", err) @@ -2044,6 +2060,7 @@ func completeAPIKeyCodexModelsManifestMetadata(body []byte, completeAll, officia return nil, fmt.Errorf("decode top-level models array: %w", err) } + officialOpenAI := account != nil && isOfficialOpenAIModelsBaseURL(account.GetOpenAIBaseURL()) changed := false if officialOpenAI { filtered := make([]json.RawMessage, 0, len(models)) @@ -2084,6 +2101,9 @@ func completeAPIKeyCodexModelsManifestMetadata(body []byte, completeAll, officia } descriptor := newConfiguredCodexModelDescriptor(slug) + if accountCodexModelSupportsImageInput(account, slug) { + descriptor.InputModalities = []string{"text", "image"} + } if forceOfficialImage { descriptor.InputModalities = []string{"text", "image"} descriptor.SupportsImageDetailOriginal = true diff --git a/backend/internal/service/openai_codex_models_service_test.go b/backend/internal/service/openai_codex_models_service_test.go index 77a65cfb38..dadf89d78e 100644 --- a/backend/internal/service/openai_codex_models_service_test.go +++ b/backend/internal/service/openai_codex_models_service_test.go @@ -492,7 +492,7 @@ func TestBuildCodexModelsManifestForGroupAdvertisesOfficialOpenAIResponsesImageI require.Equal(t, []any{"text", "image"}, models[0]["input_modalities"]) } -func TestBuildCodexModelsManifestForGroupUsesConservativeProviderImageCapabilities(t *testing.T) { +func TestBuildCodexModelsManifestForGroupUsesProviderImageCapabilities(t *testing.T) { t.Parallel() tests := []struct { @@ -602,6 +602,15 @@ func TestBuildCodexModelsManifestForGroupUsesConservativeProviderImageCapabiliti ID: 18, Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Credentials: map[string]any{"base_url": "https://openai-compatible.example.test/v1"}, }}, + modalities: []any{"text", "image"}, + }, + { + name: "custom OpenAI-compatible host with unknown model", + model: "company-coding-model", + accounts: []Account{{ + ID: 29, Platform: PlatformOpenAI, Type: AccountTypeAPIKey, + Credentials: map[string]any{"base_url": "https://openai-compatible.example.test/v1"}, + }}, modalities: []any{"text"}, }, } @@ -628,6 +637,67 @@ func TestBuildCodexModelsManifestForGroupUsesConservativeProviderImageCapabiliti } } +func TestBuildCodexModelsManifestForGroupPrefersSyncedOpenAIImageCapabilities(t *testing.T) { + t.Parallel() + + newAccount := func(id int64, modalities []string) Account { + account := Account{ + ID: id, Platform: PlatformOpenAI, Type: AccountTypeAPIKey, + Credentials: map[string]any{"base_url": "https://openai-compatible.example.test/v1"}, + } + if modalities != nil { + account.SetUpstreamModelMetadataSnapshot(UpstreamModelMetadataSnapshot{Models: map[string]UpstreamModelMetadata{ + "gpt-5.6-sol": {ID: "gpt-5.6-sol", InputModalities: modalities}, + }}) + } + return account + } + + tests := []struct { + name string + accounts []Account + modalities []any + }{ + { + name: "explicit text-only snapshot narrows local fallback", + accounts: []Account{ + newAccount(30, nil), + newAccount(31, []string{"text"}), + }, + modalities: []any{"text"}, + }, + { + name: "explicit multimodal snapshot preserves local fallback", + accounts: []Account{ + newAccount(32, nil), + newAccount(33, []string{"text", "image"}), + }, + modalities: []any{"text", "image"}, + }, + } + + for i, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + groupID := int64(760 + i) + svc := &GatewayService{accountRepo: codexModelsVisibilityAccountRepo{byGroup: map[int64][]Account{ + groupID: tt.accounts, + }}} + body, err := svc.BuildCodexModelsManifestForGroup( + context.Background(), + &Group{ID: groupID, Platform: PlatformOpenAI}, + "", + []string{"gpt-5.6-sol"}, + ) + require.NoError(t, err) + models := decodeCodexManifestModels(t, body) + require.Len(t, models, 1) + require.Equal(t, tt.modalities, models[0]["input_modalities"]) + }) + } +} + func TestBuildCodexModelsManifestForGroupUsesExplicitCompositeResponsesRouteModel(t *testing.T) { t.Parallel() @@ -1723,6 +1793,8 @@ func TestFetchCodexModelsManifestAPIKeyConvertsStandardOpenAIModelList(t *testin require.Len(t, models, 2) requireCompleteConfiguredCodexModel(t, models[0], "gpt-5.6") requireCompleteConfiguredCodexModel(t, models[1], "gpt-5.6-codex") + require.Equal(t, []any{"text", "image"}, models[0]["input_modalities"]) + require.Equal(t, []any{"text", "image"}, models[1]["input_modalities"]) require.Equal(t, codexModelsManifestBodyETag(manifest.Body), manifest.ETag) require.Equal(t, `W/"openai-list"`, manifest.upstreamETag) } @@ -1769,6 +1841,32 @@ func TestCompleteAPIKeyCodexModelsManifestForClientPreservesProviderMetadata(t * require.Equal(t, map[string]any{"source": "upstream"}, envelope["metadata"]) } +func TestCompleteAPIKeyCodexModelsManifestForClientUsesKnownGPTImageFallback(t *testing.T) { + t.Parallel() + + svc := &OpenAIGatewayService{} + manifest := &CodexModelsManifest{Body: []byte(`{"models":[ + {"slug":"gpt-5.6-sol"}, + {"slug":"company-coding-model"}, + {"slug":"gpt-4o","input_modalities":["text"]} + ]}`)} + account := newCodexModelsAPIKeyTestAccount("https://openai-compatible.example.test/v1") + + require.NoError(t, svc.CompleteAPIKeyCodexModelsManifestForClient(manifest, account)) + models := decodeCodexManifestModels(t, manifest.Body) + require.Len(t, models, 3) + + bySlug := make(map[string]map[string]any, len(models)) + for _, model := range models { + slug, ok := model["slug"].(string) + require.True(t, ok) + bySlug[slug] = model + } + require.Equal(t, []any{"text", "image"}, bySlug["gpt-5.6-sol"]["input_modalities"]) + require.Equal(t, []any{"text"}, bySlug["company-coding-model"]["input_modalities"]) + require.Equal(t, []any{"text"}, bySlug["gpt-4o"]["input_modalities"]) +} + // Scenario: 标准 /models 型号列表优先使用已同步账号能力，再使用本地 descriptor 兜底。 func TestCompleteAPIKeyCodexModelsManifestForClientUsesSyncedMetadataForConvertedModelList(t *testing.T) { t.Parallel()