diff --git a/backend/internal/service/openai_gateway_deepseek_input_image_test.go b/backend/internal/service/openai_gateway_deepseek_input_image_test.go new file mode 100644 index 0000000000..61f75c1939 --- /dev/null +++ b/backend/internal/service/openai_gateway_deepseek_input_image_test.go @@ -0,0 +1,171 @@ +package service + +import ( + "bytes" + "context" + "io" + "net/http" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/pkg/openai_compat" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +const deepSeekInputImageDataURI = "data:image/png;base64,AQID" + +func openaiMappedDeepSeekResponsesImageAccount() *Account { + return &Account{ + ID: 18, + Name: "openai-deepseek-responses", + Platform: PlatformOpenAI, + Type: AccountTypeAPIKey, + Extra: map[string]any{ + openai_compat.ExtraKeyResponsesSupported: true, + }, + Credentials: map[string]any{ + "api_key": "sk-test", + "base_url": DefaultDeepseekBaseURL, + }, + Status: StatusActive, + Schedulable: true, + } +} + +func deepSeekNativeResponsesImageAccount() *Account { + return &Account{ + Platform: PlatformDeepseek, + Type: AccountTypeAPIKey, + Credentials: map[string]any{ + "api_key": "sk-test", + "api_protocol": APIProtocolResponses, + }, + } +} + +func deepSeekUserImageBody(partJSON string) []byte { + return []byte(`{"model":"deepseek-flash","store":true,"input":[{"type":"message","role":"user","content":[{"type":"input_text","text":"what is this?"},` + partJSON + `]}]}`) +} + +func TestNormalizeDeepSeekResponsesRequestBodyAliasesInputImageURL(t *testing.T) { + t.Parallel() + + native := deepSeekNativeResponsesImageAccount() + mapped := openaiMappedDeepSeekResponsesImageAccount() + kimi := &Account{ + Platform: PlatformKimi, Type: AccountTypeAPIKey, + Credentials: map[string]any{"api_protocol": APIProtocolResponses}, + } + openai := &Account{Platform: PlatformOpenAI, Type: AccountTypeAPIKey} + + t.Run("openai_image_url_string_gets_url", func(t *testing.T) { + body := deepSeekUserImageBody(`{"type":"input_image","image_url":"` + deepSeekInputImageDataURI + `"}`) + got := normalizeDeepSeekResponsesRequestBody(mapped, body) + require.Equal(t, "input_image", gjson.GetBytes(got, "input.0.content.1.type").String()) + require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.0.content.1.image_url").String()) + require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.0.content.1.url").String()) + require.Equal(t, "what is this?", gjson.GetBytes(got, "input.0.content.0.text").String()) + require.True(t, gjson.GetBytes(got, "store").Bool(), "mapped openai 账号不应被无状态适配改掉 store") + }) + + t.Run("nested_image_url_object_flattened", func(t *testing.T) { + body := deepSeekUserImageBody(`{"type":"input_image","image_url":{"url":"` + deepSeekInputImageDataURI + `"}}`) + got := normalizeDeepSeekResponsesRequestBody(native, body) + require.Equal(t, gjson.String, gjson.GetBytes(got, "input.0.content.1.image_url").Type) + require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.0.content.1.image_url").String()) + require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.0.content.1.url").String()) + }) + + t.Run("chat_completions_image_url_part_converted", func(t *testing.T) { + body := deepSeekUserImageBody(`{"type":"image_url","image_url":{"url":"` + deepSeekInputImageDataURI + `"}}`) + got := normalizeDeepSeekResponsesRequestBody(mapped, body) + require.Equal(t, "input_image", gjson.GetBytes(got, "input.0.content.1.type").String()) + require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.0.content.1.image_url").String()) + require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.0.content.1.url").String()) + }) + + t.Run("anthropic_source_becomes_data_uri", func(t *testing.T) { + body := deepSeekUserImageBody(`{"type":"image","source":{"type":"base64","media_type":"image/png","data":"AQID"}}`) + got := normalizeDeepSeekResponsesRequestBody(native, body) + require.Equal(t, "input_image", gjson.GetBytes(got, "input.0.content.1.type").String()) + require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.0.content.1.image_url").String()) + require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.0.content.1.url").String()) + }) + + t.Run("lifted_tool_output_image_gets_url", func(t *testing.T) { + body := []byte(`{"model":"deepseek-flash","input":[{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"},{"type":"function_call_output","call_id":"call_image","output":[{"type":"input_image","image_url":"` + deepSeekInputImageDataURI + `"}]}]}`) + got := normalizeDeepSeekResponsesRequestBody(mapped, body) + require.Equal(t, gjson.String, gjson.GetBytes(got, "input.1.output").Type) + require.Equal(t, "input_image", gjson.GetBytes(got, "input.2.content.1.type").String()) + require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.2.content.1.image_url").String()) + require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.2.content.1.url").String()) + }) + + t.Run("file_id_only_does_not_gain_empty_url", func(t *testing.T) { + body := deepSeekUserImageBody(`{"type":"input_image","file_id":"file-abc"}`) + got := normalizeDeepSeekResponsesRequestBody(native, body) + require.Equal(t, "file-abc", gjson.GetBytes(got, "input.0.content.1.file_id").String()) + require.False(t, gjson.GetBytes(got, "input.0.content.1.url").Exists()) + require.False(t, gjson.GetBytes(got, "input.0.content.1.image_url").Exists()) + }) + + t.Run("kimi_does_not_alias_url", func(t *testing.T) { + body := deepSeekUserImageBody(`{"type":"input_image","image_url":"` + deepSeekInputImageDataURI + `"}`) + got := normalizeDeepSeekResponsesRequestBody(kimi, body) + require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(got, "input.0.content.1.image_url").String()) + require.False(t, gjson.GetBytes(got, "input.0.content.1.url").Exists()) + }) + + t.Run("openai_host_unchanged", func(t *testing.T) { + body := deepSeekUserImageBody(`{"type":"input_image","image_url":"` + deepSeekInputImageDataURI + `"}`) + got := normalizeDeepSeekResponsesRequestBody(openai, body) + require.Equal(t, string(body), string(got)) + require.False(t, gjson.GetBytes(got, "input.0.content.1.url").Exists()) + }) +} + +func TestForwardResponses_OpenAIMappedDeepSeekAliasesInputImageURL(t *testing.T) { + body := deepSeekUserImageBody(`{"type":"input_image","image_url":"` + deepSeekInputImageDataURI + `"}`) + c := newDeepSeekChatFallbackContext(t, body) + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader([]byte( + `{"id":"resp_ds_img","object":"response","model":"deepseek-flash","status":"completed","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"ok"}]}],"usage":{"input_tokens":5,"output_tokens":2,"total_tokens":7}}`, + ))), + }} + svc := &OpenAIGatewayService{ + cfg: deepSeekChatFallbackTestConfig(), + httpUpstream: upstream, + } + + result, err := svc.Forward(context.Background(), c, openaiMappedDeepSeekResponsesImageAccount(), body) + require.NoError(t, err) + require.NotNil(t, result) + require.Contains(t, upstream.lastReq.URL.String(), "/responses") + require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(upstream.lastBody, "input.0.content.1.image_url").String()) + require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(upstream.lastBody, "input.0.content.1.url").String()) +} + +func TestForwardResponses_NativeDeepSeekAliasesInputImageURL(t *testing.T) { + body := deepSeekUserImageBody(`{"type":"input_image","image_url":"` + deepSeekInputImageDataURI + `"}`) + c := newDeepSeekChatFallbackContext(t, body) + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader([]byte( + `{"id":"resp_ds_img","object":"response","model":"deepseek-flash","status":"completed","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"ok"}]}],"usage":{"input_tokens":5,"output_tokens":2,"total_tokens":7}}`, + ))), + }} + svc := &OpenAIGatewayService{ + cfg: deepSeekChatFallbackTestConfig(), + httpUpstream: upstream, + } + + result, err := svc.Forward(context.Background(), c, deepSeekNativeResponsesImageAccount(), body) + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "https://api.deepseek.com/responses", upstream.lastReq.URL.String()) + require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(upstream.lastBody, "input.0.content.1.image_url").String()) + require.Equal(t, deepSeekInputImageDataURI, gjson.GetBytes(upstream.lastBody, "input.0.content.1.url").String()) +} diff --git a/backend/internal/service/openai_gateway_request_body.go b/backend/internal/service/openai_gateway_request_body.go index 70c10d104a..278734a424 100644 --- a/backend/internal/service/openai_gateway_request_body.go +++ b/backend/internal/service/openai_gateway_request_body.go @@ -130,17 +130,31 @@ func deleteOpenAIResponsesNoneReasoningEffortFromObject(account *Account, body m // normalizeDeepSeekResponsesRequestBody 适配无状态 CN Responses 端点: // 强制 store=false 并清除 previous_response_id(DeepSeek / Kimi 官方 // Responses 均不支持服务端状态存储,携带这些字段会被拒绝)。 -// 非原生 Responses 协议账号原样返回。 +// +// DeepSeek 另需把 Codex/OpenAI 的 input_image.image_url 写成线上 serde +// 要求的 url 字段;openai 平台但 base_url 指向 api.deepseek.com 的映射 +// 账号同样走这条出站改写(Codex 贴图会 422 missing field url)。 +// 非 CN / 非 DeepSeek 上游原样返回。 func normalizeDeepSeekResponsesRequestBody(account *Account, body []byte) []byte { - if account == nil || !account.UsesNativeCNResponses() { + if account == nil { return body } - normalized, err := sjson.SetBytes(body, "store", false) - if err != nil { + applyStateless := account.UsesNativeCNResponses() + applyImages := shouldAliasDeepSeekResponsesInputImages(account) + if !applyStateless && !applyImages { return body } - if stripped, err := sjson.DeleteBytes(normalized, "previous_response_id"); err == nil { - normalized = stripped + + normalized := body + if applyStateless { + patched, err := sjson.SetBytes(normalized, "store", false) + if err != nil { + return body + } + normalized = patched + if stripped, err := sjson.DeleteBytes(normalized, "previous_response_id"); err == nil { + normalized = stripped + } } var requestBody map[string]any @@ -151,11 +165,22 @@ func normalizeDeepSeekResponsesRequestBody(account *Account, body []byte) []byte if !exists { return normalized } - liftedInput, changed := apicompat.LiftResponsesToolOutputMedia(input) + + changed := false + if liftedInput, lifted := apicompat.LiftResponsesToolOutputMedia(input); lifted { + requestBody["input"] = liftedInput + input = liftedInput + changed = true + } + if applyImages { + if aliased, aliasedChanged := aliasDeepSeekResponsesInputImages(input); aliasedChanged { + requestBody["input"] = aliased + changed = true + } + } if !changed { return normalized } - requestBody["input"] = liftedInput rebuilt, err := marshalOpenAIUpstreamJSON(requestBody) if err != nil { return normalized @@ -163,6 +188,162 @@ func normalizeDeepSeekResponsesRequestBody(account *Account, body []byte) []byte return rebuilt } +func shouldAliasDeepSeekResponsesInputImages(account *Account) bool { + return targetsDeepSeekAPIHost(account) +} + +// aliasDeepSeekResponsesInputImages 把图片 part 改写成 DeepSeek 能反序列化 +// 的形状:type=input_image,同时带字符串字段 image_url 与 url。 +// 文档与部分 400 提 image_url;线上 serde 要求 url。只发其中一个都会踩坑。 +// 仅有 file_id、没有可用 URL 的 part 原样保留。 +func aliasDeepSeekResponsesInputImages(input any) (any, bool) { + items, ok := asDeepSeekResponsesSlice(input) + if !ok { + return input, false + } + changed := false + for i, raw := range items { + item, ok := raw.(map[string]any) + if !ok { + continue + } + if aliasDeepSeekResponsesInputItem(item) { + items[i] = item + changed = true + } + } + if !changed { + return input, false + } + return items, true +} + +func aliasDeepSeekResponsesInputItem(item map[string]any) bool { + changed := aliasDeepSeekResponsesImagePart(item) + if content, exists := item["content"]; exists { + if rewritten, did := aliasDeepSeekResponsesContent(content); did { + item["content"] = rewritten + changed = true + } + } + if output, exists := item["output"]; exists { + if rewritten, did := aliasDeepSeekResponsesContent(output); did { + item["output"] = rewritten + changed = true + } + } + return changed +} + +func aliasDeepSeekResponsesContent(content any) (any, bool) { + parts, ok := asDeepSeekResponsesSlice(content) + if !ok { + return content, false + } + changed := false + for i, raw := range parts { + part, ok := raw.(map[string]any) + if !ok { + continue + } + if aliasDeepSeekResponsesImagePart(part) { + parts[i] = part + changed = true + } + } + if !changed { + return content, false + } + return parts, true +} + +func aliasDeepSeekResponsesImagePart(part map[string]any) bool { + partType := strings.TrimSpace(stringValue(part["type"])) + switch partType { + case "input_image", "image_url", "image": + default: + return false + } + imageURL := extractDeepSeekResponsesImageURL(part) + if imageURL == "" { + return false + } + changed := false + if partType != "input_image" { + part["type"] = "input_image" + changed = true + } + if current, ok := part["image_url"].(string); !ok || strings.TrimSpace(current) != imageURL { + part["image_url"] = imageURL + changed = true + } + if current, ok := part["url"].(string); !ok || strings.TrimSpace(current) != imageURL { + part["url"] = imageURL + changed = true + } + return changed +} + +func extractDeepSeekResponsesImageURL(part map[string]any) string { + if imageURL := deepSeekResponsesURLValue(part["url"]); imageURL != "" { + return imageURL + } + if imageURL := deepSeekResponsesURLValue(part["image_url"]); imageURL != "" { + return imageURL + } + if imageURL := deepSeekResponsesURLValue(part["image"]); imageURL != "" { + return imageURL + } + source, ok := part["source"].(map[string]any) + if !ok { + return "" + } + if imageURL := deepSeekResponsesURLValue(source["url"]); imageURL != "" { + return imageURL + } + data := strings.TrimSpace(stringValue(source["data"])) + if data == "" { + return "" + } + if strings.HasPrefix(strings.ToLower(data), "data:") { + return data + } + mediaType := strings.TrimSpace(stringValue(source["media_type"])) + if mediaType == "" { + mediaType = "image/png" + } + return "data:" + mediaType + ";base64," + data +} + +func deepSeekResponsesURLValue(value any) string { + switch typed := value.(type) { + case string: + return strings.TrimSpace(typed) + case map[string]any: + if imageURL := strings.TrimSpace(stringValue(typed["url"])); imageURL != "" { + return imageURL + } + return strings.TrimSpace(stringValue(typed["image_url"])) + default: + return "" + } +} + +func asDeepSeekResponsesSlice(value any) ([]any, bool) { + switch items := value.(type) { + case []any: + return items, true + case []map[string]any: + out := make([]any, len(items)) + for i := range items { + out[i] = items[i] + } + return out, true + default: + return nil, false + } +} + func trimOpenAIEncryptedReasoningItems(reqBody map[string]any) bool { if len(reqBody) == 0 { return false