diff --git a/backend/internal/service/openai_gateway_grok.go b/backend/internal/service/openai_gateway_grok.go index d6db84cdb4..d9c7c5b0e4 100644 --- a/backend/internal/service/openai_gateway_grok.go +++ b/backend/internal/service/openai_gateway_grok.go @@ -495,6 +495,10 @@ func patchGrokResponsesBodyBase(body []byte, upstreamModel string) ([]byte, erro if err != nil { return nil, err } + out, err = stripRedundantGrokViewImageTool(out) + if err != nil { + return nil, err + } out, err = sanitizeGrokReasoningNullContent(out) if err != nil { return nil, err @@ -767,6 +771,69 @@ func sanitizeGrokResponsesInput(body []byte) ([]byte, error) { return sjson.SetRawBytes(body, "tools", encodedTools) } +// An inline input_image is already visible to Grok. Keeping Codex's local +// view_image tool in the same turn can make Grok announce a tool call without +// actually calling it, so remove only that redundant automatic choice. +func stripRedundantGrokViewImageTool(body []byte) ([]byte, error) { + input := gjson.GetBytes(body, "input") + if !input.IsArray() { + return body, nil + } + items := input.Array() + if len(items) == 0 { + return body, nil + } + current := items[len(items)-1] + if strings.TrimSpace(current.Get("role").String()) != "user" || + !openAIJSONValueMayContainImageInput(current) { + return body, nil + } + + toolChoice := gjson.GetBytes(body, "tool_choice") + if toolChoice.IsObject() && strings.TrimSpace(toolChoice.Get("type").String()) == "function" { + choiceName := strings.TrimSpace(toolChoice.Get("name").String()) + if choiceName == "" { + choiceName = strings.TrimSpace(toolChoice.Get("function.name").String()) + } + if choiceName == "view_image" { + return body, nil + } + } + + tools := gjson.GetBytes(body, "tools") + if !tools.IsArray() { + return body, nil + } + filtered := make([]json.RawMessage, 0, len(tools.Array())) + changed := false + for _, tool := range tools.Array() { + if strings.TrimSpace(tool.Get("type").String()) == "function" && + strings.TrimSpace(tool.Get("name").String()) == "view_image" { + changed = true + continue + } + filtered = append(filtered, json.RawMessage(tool.Raw)) + } + if !changed { + return body, nil + } + if len(filtered) == 0 && strings.TrimSpace(toolChoice.String()) == "required" { + return body, nil + } + + if len(filtered) == 0 { + out, err := sjson.DeleteBytes(body, "tools") + if err != nil { + return nil, err + } + return sjson.DeleteBytes(out, "parallel_tool_calls") + } + encoded, err := json.Marshal(filtered) + if err != nil { + return nil, err + } + return sjson.SetRawBytes(body, "tools", encoded) +} func grokResponsesToolDedupKey(tool gjson.Result) string { toolType := strings.TrimSpace(tool.Get("type").String()) if toolType != "" { diff --git a/backend/internal/service/openai_gateway_grok_test.go b/backend/internal/service/openai_gateway_grok_test.go index 8e34ec85db..7b30e683aa 100644 --- a/backend/internal/service/openai_gateway_grok_test.go +++ b/backend/internal/service/openai_gateway_grok_test.go @@ -43,6 +43,107 @@ func TestPatchGrokResponsesBodySetsMappedModelAndDropsUnsupportedFields(t *testi require.Equal(t, "high", gjson.GetBytes(patched, "reasoning.effort").String()) } +func TestPatchGrokResponsesBodyDropsRedundantViewImageForCurrentInlineImage(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + body string + }{ + { + name: "top-level tools", + body: `{ + "model":"grok-4.6", + "input":[{"type":"message","role":"user","content":[ + {"type":"input_text","text":"What text is in this image?"}, + {"type":"input_image","image_url":"data:image/png;base64,AA=="} + ]}], + "tools":[ + {"type":"function","name":"view_image","parameters":{"type":"object"}}, + {"type":"function","name":"shell_command","parameters":{"type":"object"}} + ] + }`, + }, + { + name: "Responses Lite additional tools", + body: `{ + "model":"grok-4.6", + "input":[ + {"type":"additional_tools","role":"developer","tools":[ + {"type":"function","name":"view_image","parameters":{"type":"object"}}, + {"type":"function","name":"shell_command","parameters":{"type":"object"}} + ]}, + {"type":"message","role":"user","content":[ + {"type":"input_text","text":"What text is in this image?"}, + {"type":"input_image","image_url":"data:image/png;base64,AA=="} + ]} + ] + }`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + patched, err := patchGrokResponsesBody([]byte(tt.body), "grok-4.6") + require.NoError(t, err) + require.False(t, gjson.GetBytes(patched, `tools.#(name=="view_image")`).Exists()) + require.Equal(t, "shell_command", gjson.GetBytes(patched, "tools.0.name").String()) + }) + } +} + +func TestPatchGrokResponsesBodyKeepsNonRedundantViewImage(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + body string + }{ + { + name: "current turn has no inline image", + body: `{"input":[{"role":"user","content":[{"type":"input_text","text":"Inspect a local image"}]}],"tools":[{"type":"function","name":"view_image"}]}`, + }, + { + name: "inline image is only historical", + body: `{"input":[{"role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,AA=="}]},{"role":"assistant","content":[{"type":"output_text","text":"Done"}]},{"role":"user","content":[{"type":"input_text","text":"Inspect another local image"}]}],"tools":[{"type":"function","name":"view_image"}]}`, + }, + { + name: "view image is explicitly selected", + body: `{"input":[{"role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,AA=="}]}],"tools":[{"type":"function","name":"view_image"}],"tool_choice":{"type":"function","name":"view_image"}}`, + }, + { + name: "required with view image as the only tool", + body: `{"input":[{"role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,AA=="}]}],"tools":[{"type":"function","name":"view_image"}],"tool_choice":"required"}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + patched, err := patchGrokResponsesBody([]byte(tt.body), "grok-4.6") + require.NoError(t, err) + require.Equal(t, "view_image", gjson.GetBytes(patched, "tools.0.name").String()) + }) + } +} + +func TestPatchGrokResponsesBodyDropsViewImageOnlyToolMetadata(t *testing.T) { + t.Parallel() + + body := []byte(`{ + "input":[{"role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,AA=="}]}], + "tools":[{"type":"function","name":"view_image"}], + "tool_choice":"auto", + "parallel_tool_calls":true + }`) + patched, err := patchGrokResponsesBody(body, "grok-4.6") + require.NoError(t, err) + require.False(t, gjson.GetBytes(patched, "tools").Exists()) + require.False(t, gjson.GetBytes(patched, "tool_choice").Exists()) + require.False(t, gjson.GetBytes(patched, "parallel_tool_calls").Exists()) +} + func TestPatchGrokResponsesBodySanitizesComposerReasoningParameters(t *testing.T) { t.Parallel()