From b0cdea303afd81071f0b0ccc762cb5141480133a Mon Sep 17 00:00:00 2001 From: lyen1688 Date: Wed, 19 Aug 2026 19:04:53 +0800 Subject: [PATCH] =?UTF-8?q?=E8=A1=A5=E5=85=A8=20Grok=20=E5=A4=9A=E5=85=A5?= =?UTF-8?q?=E5=8F=A3=E5=86=85=E8=81=94=E5=9B=BE=E7=89=87=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E9=80=82=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit raw Chat 绕过 Responses 公共补丁，需按 Chat schema 单独移除当前轮内联图片对应的冗余 view_image；同时补充 Chat、raw Chat 与 Messages 真实出站回归测试。 --- .../openai_gateway_chat_completions_raw.go | 7 + .../openai_gateway_grok_chat_image_tools.go | 83 ++++++++ ...nai_gateway_grok_inline_image_tool_test.go | 184 ++++++++++++++++++ 3 files changed, 274 insertions(+) create mode 100644 backend/internal/service/openai_gateway_grok_chat_image_tools.go create mode 100644 backend/internal/service/openai_gateway_grok_inline_image_tool_test.go diff --git a/backend/internal/service/openai_gateway_chat_completions_raw.go b/backend/internal/service/openai_gateway_chat_completions_raw.go index 34c281b28e..3c46194188 100644 --- a/backend/internal/service/openai_gateway_chat_completions_raw.go +++ b/backend/internal/service/openai_gateway_chat_completions_raw.go @@ -105,6 +105,13 @@ func (s *OpenAIGatewayService) forwardAsRawChatCompletions( return nil, policyErr } upstreamBody = updatedBody + if account.Platform == PlatformGrok { + strippedBody, stripErr := stripRedundantGrokChatViewImageTool(upstreamBody) + if stripErr != nil { + return nil, fmt.Errorf("strip redundant Grok Chat view_image tool: %w", stripErr) + } + upstreamBody = strippedBody + } // Grok Composer does not accept image_url parts directly, but Grok Build // can describe the images first. Bridge only this exact failure mode. diff --git a/backend/internal/service/openai_gateway_grok_chat_image_tools.go b/backend/internal/service/openai_gateway_grok_chat_image_tools.go new file mode 100644 index 0000000000..2e15faa21f --- /dev/null +++ b/backend/internal/service/openai_gateway_grok_chat_image_tools.go @@ -0,0 +1,83 @@ +package service + +import ( + "encoding/json" + "strings" + + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" +) + +// 当前轮的内联 image_url 已可由 Grok 直接读取；若同时保留客户端本地 +// view_image，Grok 可能只预告调用工具而不继续作答，因此只移除这一冗余自动选择。 +func stripRedundantGrokChatViewImageTool(body []byte) ([]byte, error) { + messages := gjson.GetBytes(body, "messages") + if !messages.IsArray() { + return body, nil + } + items := messages.Array() + if len(items) == 0 { + return body, nil + } + current := items[len(items)-1] + if strings.TrimSpace(current.Get("role").String()) != "user" || + !openAIJSONValueMayContainImageInput(current) { + return body, nil + } + + toolChoice := gjson.GetBytes(body, "tool_choice") + if toolChoice.IsObject() && strings.TrimSpace(toolChoice.Get("type").String()) == "function" { + choiceName := strings.TrimSpace(toolChoice.Get("function.name").String()) + if choiceName == "" { + choiceName = strings.TrimSpace(toolChoice.Get("name").String()) + } + if choiceName == "view_image" { + return body, nil + } + } + + tools := gjson.GetBytes(body, "tools") + if !tools.IsArray() { + return body, nil + } + filtered := make([]json.RawMessage, 0, len(tools.Array())) + changed := false + for _, tool := range tools.Array() { + toolName := strings.TrimSpace(tool.Get("function.name").String()) + if toolName == "" { + toolName = strings.TrimSpace(tool.Get("name").String()) + } + if strings.TrimSpace(tool.Get("type").String()) == "function" && toolName == "view_image" { + changed = true + continue + } + filtered = append(filtered, json.RawMessage(tool.Raw)) + } + if !changed { + return body, nil + } + if len(filtered) == 0 && strings.TrimSpace(toolChoice.String()) == "required" { + return body, nil + } + + if len(filtered) > 0 { + encoded, err := json.Marshal(filtered) + if err != nil { + return nil, err + } + return sjson.SetRawBytes(body, "tools", encoded) + } + + out, err := sjson.DeleteBytes(body, "tools") + if err != nil { + return nil, err + } + out, err = sjson.DeleteBytes(out, "parallel_tool_calls") + if err != nil { + return nil, err + } + if strings.TrimSpace(toolChoice.String()) == "auto" { + out, err = sjson.DeleteBytes(out, "tool_choice") + } + return out, err +} diff --git a/backend/internal/service/openai_gateway_grok_inline_image_tool_test.go b/backend/internal/service/openai_gateway_grok_inline_image_tool_test.go new file mode 100644 index 0000000000..51df1823cf --- /dev/null +++ b/backend/internal/service/openai_gateway_grok_inline_image_tool_test.go @@ -0,0 +1,184 @@ +//go:build unit + +package service + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/pkg/xai" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +func TestForwardGrokChatViaResponsesDropsRedundantViewImage(t *testing.T) { + gin.SetMode(gin.TestMode) + body := grokChatInlineImageRequest() + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(body)) + c.Set("api_key", &APIKey{ID: 7991}) + + account := grokChatBridgeTestAccount(799) + repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{ + accountsByID: map[int64]*Account{account.ID: account}, + }} + upstream := &httpUpstreamRecorder{resp: grokChatBridgeCompletedResponse("resp_chat_image", 0)} + svc := &OpenAIGatewayService{ + httpUpstream: upstream, + grokTokenProvider: NewGrokTokenProvider(repo, nil), + accountRepo: repo, + } + + result, err := svc.ForwardAsChatCompletions(context.Background(), c, account, body, "", "") + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, xai.DefaultCLIBaseURL+"/responses", upstream.lastReq.URL.String()) + require.Equal(t, "input_image", gjson.GetBytes(upstream.lastBody, "input.0.content.1.type").String()) + assertGrokUpstreamKeepsOtherToolAndDropsViewImage(t, upstream.lastBody, "tools.#(name==\"%s\")") +} + +func TestForwardGrokRawChatDropsRedundantViewImage(t *testing.T) { + gin.SetMode(gin.TestMode) + body := grokChatInlineImageRequest() + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(body)) + + account := &Account{ + ID: 800, Platform: PlatformGrok, Type: AccountTypeAPIKey, Concurrency: 1, + Credentials: map[string]any{"api_key": "test-key", "base_url": "https://grok.example.test/v1"}, + } + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader( + `{"id":"chatcmpl","object":"chat.completion","model":"grok-4.6","choices":[],"usage":{"prompt_tokens":1,"completion_tokens":1}}`, + )), + }} + svc := &OpenAIGatewayService{cfg: rawChatCompletionsTestConfig(), httpUpstream: upstream} + + result, err := svc.ForwardAsChatCompletions(context.Background(), c, account, body, "", "") + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "https://grok.example.test/v1/chat/completions", upstream.lastReq.URL.String()) + require.Equal(t, "image_url", gjson.GetBytes(upstream.lastBody, "messages.0.content.1.type").String()) + assertGrokUpstreamKeepsOtherToolAndDropsViewImage(t, upstream.lastBody, "tools.#(function.name==\"%s\")") +} + +func TestForwardGrokMessagesDropsRedundantViewImage(t *testing.T) { + gin.SetMode(gin.TestMode) + body := []byte(`{ + "model":"grok-4.6","max_tokens":32,"stream":false, + "messages":[{"role":"user","content":[ + {"type":"text","text":"What text is in this image?"}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}} + ]}], + "tools":[ + {"name":"view_image","input_schema":{"type":"object","properties":{"path":{"type":"string"}}}}, + {"name":"shell_command","input_schema":{"type":"object","properties":{"cmd":{"type":"string"}}}} + ], + "tool_choice":{"type":"auto"} + }`) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", bytes.NewReader(body)) + c.Set("api_key", &APIKey{ID: 7992}) + + account := healthyGrokOAuthGatewayTestAccount(801, "access-token") + repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{ + accountsByID: map[int64]*Account{account.ID: account}, + }} + upstream := &httpUpstreamRecorder{resp: grokMessagesSSECompletedResponse("resp_messages_image", 0)} + svc := &OpenAIGatewayService{ + httpUpstream: upstream, + grokTokenProvider: NewGrokTokenProvider(repo, nil), + accountRepo: repo, + } + + result, err := svc.ForwardAsAnthropic(context.Background(), c, account, body, "", "") + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, xai.DefaultCLIBaseURL+"/responses", upstream.lastReq.URL.String()) + require.Equal(t, "input_image", gjson.GetBytes(upstream.lastBody, "input.0.content.1.type").String()) + assertGrokUpstreamKeepsOtherToolAndDropsViewImage(t, upstream.lastBody, "tools.#(name==\"%s\")") +} + +func TestStripRedundantGrokChatViewImageToolLeavesNonTargetRequestsByteExact(t *testing.T) { + t.Parallel() + tests := []struct { + name string + body string + }{ + { + name: "current turn has no inline image", + body: `{"messages":[{"role":"user","content":"Inspect a local image"}],"tools":[{"type":"function","function":{"name":"view_image"}}]}`, + }, + { + name: "inline image is only historical", + body: `{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,AA=="}}]},{"role":"assistant","content":"Done"},{"role":"user","content":"Inspect another local image"}],"tools":[{"type":"function","function":{"name":"view_image"}}]}`, + }, + { + name: "view image is explicitly selected", + body: `{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,AA=="}}]}],"tools":[{"type":"function","function":{"name":"view_image"}}],"tool_choice":{"type":"function","function":{"name":"view_image"}}}`, + }, + { + name: "required with view image as the only tool", + body: `{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,AA=="}}]}],"tools":[{"type":"function","function":{"name":"view_image"}}],"tool_choice":"required"}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + body := []byte(tt.body) + patched, err := stripRedundantGrokChatViewImageTool(body) + require.NoError(t, err) + require.Equal(t, body, patched) + }) + } +} + +func TestStripRedundantGrokChatViewImageToolDropsOnlyToolMetadata(t *testing.T) { + t.Parallel() + body := []byte(`{ + "messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,AA=="}}]}], + "tools":[{"type":"function","function":{"name":"view_image"}}], + "tool_choice":"auto", + "parallel_tool_calls":true + }`) + + patched, err := stripRedundantGrokChatViewImageTool(body) + require.NoError(t, err) + require.False(t, gjson.GetBytes(patched, "tools").Exists()) + require.False(t, gjson.GetBytes(patched, "tool_choice").Exists()) + require.False(t, gjson.GetBytes(patched, "parallel_tool_calls").Exists()) +} + +func grokChatInlineImageRequest() []byte { + return []byte(`{ + "model":"grok-4.6", + "messages":[{"role":"user","content":[ + {"type":"text","text":"What text is in this image?"}, + {"type":"image_url","image_url":{"url":"data:image/png;base64,AA=="}} + ]}], + "stream":false, + "tools":[ + {"type":"function","function":{"name":"view_image","parameters":{"type":"object","properties":{"path":{"type":"string"}}}}}, + {"type":"function","function":{"name":"shell_command","parameters":{"type":"object","properties":{"cmd":{"type":"string"}}}}} + ], + "tool_choice":"auto" + }`) +} + +func assertGrokUpstreamKeepsOtherToolAndDropsViewImage(t *testing.T, body []byte, pathTemplate string) { + t.Helper() + require.False(t, gjson.GetBytes(body, strings.Replace(pathTemplate, "%s", "view_image", 1)).Exists(), string(body)) + require.True(t, gjson.GetBytes(body, strings.Replace(pathTemplate, "%s", "shell_command", 1)).Exists(), string(body)) +}