From 37db8d031bd7aa18905d76f7b5cb19c34d76245f Mon Sep 17 00:00:00 2001 From: haruka <1628615876@qq.com> Date: Sat, 18 Jul 2026 05:24:32 -0700 Subject: [PATCH 01/59] =?UTF-8?q?fix(config):=20=E8=AE=A9=E7=8E=AF?= =?UTF-8?q?=E5=A2=83=E5=8F=98=E9=87=8F=E8=83=BD=E7=9C=9F=E6=AD=A3=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=20image=5Fstorage=20=E7=AD=89=E5=87=AD=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit viper.Unmarshal 只解码 AllKeys() 返回的键，而 AllKeys() 只汇总 SetDefault、 配置文件和显式 BindEnv 三个来源。AutomaticEnv 仅能覆盖已在其中的键，无法引入 新键；能兜底的 viper_bind_struct 又被 build tag 排除（我们只用 -tags embed）。 因此任何「没有注册默认值、且不在 config.yaml 里」的配置项，其环境变量会被静默 丢弃。image_storage 的 endpoint/bucket/access_key_id/secret_access_key/ public_base_url 正属此列，于是纯环境变量部署落到最坏组合：IMAGE_STORAGE_ENABLED 生效使 Enabled=true，四个凭证却为空 → Active()=false → 异步生图接口整体 404， 运维看到的却是"凭证不完整"。deploy/docker-compose.yml 默认就是纯环境变量驱动， 且自动生成的 config.yaml 从不写 image_storage 段，必然踩中（见 #4458、#4542）。 同类缺口不止于此：github_oauth、google_oauth、dingtalk_connect 三组第三方登录 配置（含 client_secret）同样完全无法用环境变量设置。 - 为这些键注册零值默认，使其进入 AllKeys() 而可被环境变量覆盖。零值与"键缺失" 时的解码结果一致，故行为不变。 - sticky_escape_enabled 例外：它的实际默认是 true（靠 IsSet 守卫在解码后补上）， 注册 false 会让 IsSet 恒真而永久关闭该特性，故直接注册 true。 - 启动告警补上 missing_keys 字段，指明到底哪个凭证为空。 - 新增反射守卫测试：Config 结构体上每个可由环境变量表达的字段都必须已注册默认值。 Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01VHreE5pzCkSYz7J45fmd2Y --- backend/internal/config/config.go | 98 +++++++++++++++++++ .../internal/config/env_reachability_test.go | 90 +++++++++++++++++ .../internal/config/image_storage_env_test.go | 41 ++++++++ backend/internal/service/wire.go | 6 +- docs/ASYNC_IMAGE_TASKS.md | 16 +++ 5 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 backend/internal/config/env_reachability_test.go create mode 100644 backend/internal/config/image_storage_env_test.go diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index def68f3a2a..2db2418c46 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -255,6 +255,22 @@ func (c *ImageStorageConfig) Active() bool { return c.Enabled && c.IsConfigured() } +// MissingCredentialKeys 返回 IsConfigured 所缺的配置键名。 +// 用于启动日志：只说"凭证不完整"会让运维以为自己漏填了，而实际可能是值填了却没被读到。 +func (c *ImageStorageConfig) MissingCredentialKeys() []string { + var missing []string + if c.Bucket == "" { + missing = append(missing, "image_storage.bucket") + } + if c.AccessKeyID == "" { + missing = append(missing, "image_storage.access_key_id") + } + if c.SecretAccessKey == "" { + missing = append(missing, "image_storage.secret_access_key") + } + return missing +} + type LinuxDoConnectConfig struct { Enabled bool `mapstructure:"enabled"` ClientID string `mapstructure:"client_id"` @@ -1562,6 +1578,9 @@ func load(allowMissingJWTSecret bool) (*Config, error) { if cfg.Gateway.OpenAIScheduler.StickyEscapeErrorRate == 0 { cfg.Gateway.OpenAIScheduler.StickyEscapeErrorRate = 0.5 } + // Kept as a backstop: setEnvReachableDefaults now registers this key with its + // effective default (true), so IsSet always reports true and this branch no + // longer fires. It still guards the default if that registration is dropped. if !cfg.Gateway.OpenAIScheduler.StickyEscapeEnabled && !viper.IsSet("gateway.openai_scheduler.sticky_escape_enabled") { cfg.Gateway.OpenAIScheduler.StickyEscapeEnabled = true } @@ -1926,6 +1945,15 @@ func setDefaults() { viper.SetDefault("image_storage.force_path_style", false) viper.SetDefault("image_storage.presign_expiry_hours", 24) viper.SetDefault("image_storage.max_download_bytes", 33554432) + // Registered with empty defaults so AutomaticEnv can reach them: viper only + // decodes keys present in AllKeys(), so a credential that is supplied purely + // via IMAGE_STORAGE_* and never appears in config.yaml would be dropped and + // silently disable the whole async image feature. + viper.SetDefault("image_storage.endpoint", "") + viper.SetDefault("image_storage.bucket", "") + viper.SetDefault("image_storage.access_key_id", "") + viper.SetDefault("image_storage.secret_access_key", "") + viper.SetDefault("image_storage.public_base_url", "") // Ops (vNext) viper.SetDefault("ops.enabled", true) @@ -2205,6 +2233,76 @@ func setDefaults() { viper.SetDefault("subscription_maintenance.worker_count", 2) viper.SetDefault("subscription_maintenance.queue_size", 1024) + setEnvReachableDefaults() +} + +// setEnvReachableDefaults registers zero-valued defaults for keys that are +// documented in deploy/config.example.yaml but had no default of their own. +// +// viper.Unmarshal only decodes the keys returned by AllKeys(), which unions +// SetDefault keys, config-file keys and explicitly bound BindEnv keys. +// AutomaticEnv can override a key already in that union, but it never adds one, +// and the viper_bind_struct escape hatch is compiled out (we build with +// -tags embed). So a key that lives only in the example file was unreachable by +// environment variable: the value was read from the process environment and +// then silently dropped. Deployments driven purely by env — which is what +// deploy/docker-compose.yml does — got the zero value with no warning. +// +// The values below are deliberately zero rather than the documented example +// values: an absent key already unmarshalled to the zero value, so registering +// zero keeps behavior identical while making the key addressable from the +// environment. Any subsystem that wants a richer default still applies it after +// unmarshal, exactly as before. +func setEnvReachableDefaults() { + viper.SetDefault("gateway.forced_codex_instructions_template_file", "") + viper.SetDefault("gateway.session_idle_timeout_minutes", 0) + viper.SetDefault("gateway.user_message_queue.mode", "") + viper.SetDefault("update.proxy_url", "") + + // sticky_escape_enabled is the one exception to the zero-value rule: its + // effective default is true, applied post-unmarshal via a viper.IsSet guard. + // Registering false would make IsSet always report true and permanently + // disable sticky escape, so register the effective default instead. An + // explicit false in config or env still wins. + viper.SetDefault("gateway.openai_scheduler.sticky_escape_enabled", true) + viper.SetDefault("gateway.openai_scheduler.sticky_escape_error_rate", 0.0) + viper.SetDefault("gateway.openai_scheduler.sticky_escape_ttft_ms", 0) + + // Third-party login providers. These carry client secrets and are exactly + // the settings an operator expects to inject via the environment, but every + // key here was previously unreachable that way. + for _, provider := range []string{"github_oauth", "google_oauth"} { + viper.SetDefault(provider+".enabled", false) + viper.SetDefault(provider+".client_id", "") + viper.SetDefault(provider+".client_secret", "") + viper.SetDefault(provider+".authorize_url", "") + viper.SetDefault(provider+".token_url", "") + viper.SetDefault(provider+".userinfo_url", "") + viper.SetDefault(provider+".emails_url", "") + viper.SetDefault(provider+".scopes", "") + viper.SetDefault(provider+".redirect_url", "") + viper.SetDefault(provider+".frontend_redirect_url", "") + } + + viper.SetDefault("dingtalk_connect.client_id", "") + viper.SetDefault("dingtalk_connect.client_secret", "") + viper.SetDefault("dingtalk_connect.internal_corp_id", "") + viper.SetDefault("dingtalk_connect.redirect_url", "") + viper.SetDefault("dingtalk_connect.bypass_registration", false) + viper.SetDefault("dingtalk_connect.username_attribute_key", "") + viper.SetDefault("dingtalk_connect.enable_attribute_matching", false) + viper.SetDefault("dingtalk_connect.enable_attribute_sync", false) + viper.SetDefault("dingtalk_connect.attribute_sync_fields", []string{}) + viper.SetDefault("dingtalk_connect.attribute_sync_overwrite_policy", "") + viper.SetDefault("dingtalk_connect.sync_display_name", false) + viper.SetDefault("dingtalk_connect.sync_display_name_attr_key", "") + viper.SetDefault("dingtalk_connect.sync_display_name_attr_name", "") + viper.SetDefault("dingtalk_connect.sync_dept", false) + viper.SetDefault("dingtalk_connect.sync_dept_attr_key", "") + viper.SetDefault("dingtalk_connect.sync_dept_attr_name", "") + viper.SetDefault("dingtalk_connect.sync_corp_email", false) + viper.SetDefault("dingtalk_connect.sync_corp_email_attr_key", "") + viper.SetDefault("dingtalk_connect.sync_corp_email_attr_name", "") } func (c *Config) Validate() error { diff --git a/backend/internal/config/env_reachability_test.go b/backend/internal/config/env_reachability_test.go new file mode 100644 index 0000000000..8141a2ec3d --- /dev/null +++ b/backend/internal/config/env_reachability_test.go @@ -0,0 +1,90 @@ +//go:build unit + +package config + +import ( + "reflect" + "sort" + "strings" + "testing" + + "github.com/spf13/viper" +) + +// collectMapstructureKeys walks a config struct and returns every dotted key +// viper would need in order to populate it. +func collectMapstructureKeys(t reflect.Type, prefix string, out map[string]string) { + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + if field.PkgPath != "" { + continue // unexported + } + tag := field.Tag.Get("mapstructure") + name, _, _ := strings.Cut(tag, ",") + if name == "-" { + continue + } + if name == "" { + name = strings.ToLower(field.Name) + } + key := name + if prefix != "" { + key = prefix + "." + name + } + + ft := field.Type + for ft.Kind() == reflect.Ptr { + ft = ft.Elem() + } + if ft.Kind() == reflect.Struct { + collectMapstructureKeys(ft, key, out) + continue + } + if ft.Kind() == reflect.Map { + // A map cannot be expressed in a single environment variable, so it + // is out of scope here — such settings need a config file either way. + continue + } + out[strings.ToLower(key)] = ft.String() + } +} + +// TestConfigKeysAreEnvReachable is the systemic guard behind the image_storage +// bug: viper.Unmarshal only decodes keys returned by AllKeys(), which unions +// SetDefault keys, config-file keys and explicit BindEnv keys. AutomaticEnv can +// override a key already in that union but never introduces one, and the +// viper_bind_struct escape hatch is compiled out (we build with -tags embed). +// +// So a Config field with no registered default is unreachable by environment +// variable whenever the deployment has no config.yaml containing it — the +// operator sets the variable, the loader discards it, and the feature behaves +// as if it were never configured. That is exactly how image_storage credentials +// were lost, silently disabling async image tasks for env-driven deployments. +// +// When this fails, register a zero-valued default in setEnvReachableDefaults +// for each reported key. +func TestConfigKeysAreEnvReachable(t *testing.T) { + bound := map[string]string{} + collectMapstructureKeys(reflect.TypeOf(Config{}), "", bound) + + viper.Reset() + t.Cleanup(viper.Reset) + setDefaults() + registered := map[string]struct{}{} + for _, key := range viper.AllKeys() { + registered[key] = struct{}{} + } + + var unreachable []string + for key, kind := range bound { + if _, ok := registered[key]; !ok { + unreachable = append(unreachable, key+" ("+kind+")") + } + } + sort.Strings(unreachable) + + if len(unreachable) > 0 { + t.Fatalf("%d config keys have no default registered, so their environment variables are silently ignored:\n %s", + len(unreachable), strings.Join(unreachable, "\n ")) + } +} diff --git a/backend/internal/config/image_storage_env_test.go b/backend/internal/config/image_storage_env_test.go new file mode 100644 index 0000000000..8061ee2ee7 --- /dev/null +++ b/backend/internal/config/image_storage_env_test.go @@ -0,0 +1,41 @@ +//go:build unit + +package config + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestLoadImageStorageFromEnv guards against a viper trap that silently disabled +// asynchronous image tasks for every environment-variable-only deployment. +// +// viper only decodes keys returned by AllKeys(), which unions SetDefault keys, +// config-file keys and explicit BindEnv keys. AutomaticEnv can override a key +// that is already in that list, but it never introduces a new one. Credentials +// such as image_storage.bucket therefore need an (empty) default registered, or +// IMAGE_STORAGE_BUCKET is dropped on the floor and Active() stays false while +// image_storage.enabled reads true — the endpoints 404 with no useful signal. +func TestLoadImageStorageFromEnv(t *testing.T) { + resetViperWithJWTSecret(t) + t.Setenv("IMAGE_STORAGE_ENABLED", "true") + t.Setenv("IMAGE_STORAGE_ENDPOINT", "https://acct.r2.cloudflarestorage.com") + t.Setenv("IMAGE_STORAGE_BUCKET", "my-images") + t.Setenv("IMAGE_STORAGE_ACCESS_KEY_ID", "ak") + t.Setenv("IMAGE_STORAGE_SECRET_ACCESS_KEY", "sk") + t.Setenv("IMAGE_STORAGE_PUBLIC_BASE_URL", "https://cdn.example.com") + + cfg, err := Load() + require.NoError(t, err) + + require.True(t, cfg.ImageStorage.Enabled) + require.Equal(t, "https://acct.r2.cloudflarestorage.com", cfg.ImageStorage.Endpoint) + require.Equal(t, "my-images", cfg.ImageStorage.Bucket) + require.Equal(t, "ak", cfg.ImageStorage.AccessKeyID) + require.Equal(t, "sk", cfg.ImageStorage.SecretAccessKey) + require.Equal(t, "https://cdn.example.com", cfg.ImageStorage.PublicBaseURL) + + require.True(t, cfg.ImageStorage.IsConfigured()) + require.True(t, cfg.ImageStorage.Active(), "async image tasks must be active when every credential is supplied via env") +} diff --git a/backend/internal/service/wire.go b/backend/internal/service/wire.go index 59037d1882..8c6abd8997 100644 --- a/backend/internal/service/wire.go +++ b/backend/internal/service/wire.go @@ -12,6 +12,7 @@ import ( "github.com/Wei-Shaw/sub2api/internal/pkg/logger" "github.com/google/wire" "github.com/redis/go-redis/v9" + "go.uber.org/zap" ) // BuildInfo contains build information @@ -529,7 +530,10 @@ func ProvideAPIKeyAuthCacheInvalidator(apiKeyService *APIKeyService) APIKeyAuthC func ProvideImageTaskService(store ImageTaskStore, storage ImageStorage, cfg *config.Config) *ImageTaskService { if !cfg.ImageStorage.Active() { if cfg.ImageStorage.Enabled { - logger.L().Warn("image_storage.enabled is true but object storage is not fully configured; async image tasks are disabled") + // 列出具体缺失的键。若这些键其实已在环境变量里设过，说明它们没被读进来， + // 请确认 setDefaults 中已为其注册默认值（见 config.setEnvReachableDefaults）。 + logger.L().Warn("image_storage.enabled is true but object storage is not fully configured; async image tasks are disabled", + zap.Strings("missing_keys", cfg.ImageStorage.MissingCredentialKeys())) } return NewImageTaskService(store) } diff --git a/docs/ASYNC_IMAGE_TASKS.md b/docs/ASYNC_IMAGE_TASKS.md index 4c6b744e92..12793f2a78 100644 --- a/docs/ASYNC_IMAGE_TASKS.md +++ b/docs/ASYNC_IMAGE_TASKS.md @@ -41,6 +41,22 @@ When a task completes, each generated image is uploaded to the bucket and the re To support a different vendor beyond the S3-compatible client, implement the `service.ImageStorage` interface (`Save(ctx, key, contentType, data) (url, error)`) and provide it in place of the S3 implementation. +### Troubleshooting: the endpoints return 404 after enabling + +`404 async image tasks are not enabled` means `image_storage` did not resolve to a complete configuration, so the feature stayed off. The route exists either way — the 404 comes from the handler, not from an unregistered path, which makes it easy to mistake for a missing build. + +Check the startup log for: + +```text +WARN image_storage.enabled is true but object storage is not fully configured; async image tasks are disabled missing_keys=[...] +``` + +`missing_keys` names exactly which credentials were empty when the config was loaded. + +Note that releases **before v0.1.161 silently dropped `IMAGE_STORAGE_ENDPOINT`, `_BUCKET`, `_ACCESS_KEY_ID`, `_SECRET_ACCESS_KEY` and `_PUBLIC_BASE_URL`** when they were supplied only through the environment: those keys had no registered default, and viper cannot see an environment variable for a key it does not already know about. Deployments driven purely by `environment:` — which is what `deploy/docker-compose.yml` does by default — therefore reported `enabled: true` with empty credentials and 404'd on every async call. On an affected release the workaround is to also place the `image_storage` block in `/app/data/config.yaml` (copy it from `deploy/config.example.yaml`); once the keys exist in the file, the environment overrides apply normally. + +Two further causes of a 404 that are unrelated to storage: the API key's group must be on the **OpenAI or Grok** platform (any other platform, or a key with no group at all, yields `Images API is not supported for this platform`), and a task may only be polled with the **same API key that submitted it** — polling with a different key of the same user returns `image task not found` by design. + ## Submit a task ```bash From 510ee451bd9e42682838972f64bd1faf027bd244 Mon Sep 17 00:00:00 2001 From: wucm667 Date: Sun, 19 Jul 2026 11:02:56 +0800 Subject: [PATCH 02/59] feat: support GitHub token for update checks --- .../repository/github_release_service.go | 51 +++++++-- .../repository/github_release_service_test.go | 92 ++++++++++++++++ deploy/.env.example | 4 + deploy/README.md | 1 + deploy/docker-compose.local.yml | 1 + deploy/docker-compose.standalone.yml | 1 + deploy/docker-compose.yml | 1 + deploy/install.sh | 54 ++++++++- deploy/tests/install-github-token-test.sh | 103 ++++++++++++++++++ 9 files changed, 298 insertions(+), 10 deletions(-) create mode 100755 deploy/tests/install-github-token-test.sh diff --git a/backend/internal/repository/github_release_service.go b/backend/internal/repository/github_release_service.go index 85fab2b344..282a215361 100644 --- a/backend/internal/repository/github_release_service.go +++ b/backend/internal/repository/github_release_service.go @@ -7,6 +7,7 @@ import ( "io" "log/slog" "net/http" + "net/url" "os" "strings" "time" @@ -18,6 +19,7 @@ import ( type githubReleaseClient struct { httpClient *http.Client downloadHTTPClient *http.Client + updateGitHubToken string } type githubReleaseClientError struct { @@ -43,6 +45,8 @@ func NewGitHubReleaseClient(proxyURL string, allowDirectOnProxyError bool) servi } sharedClient = &http.Client{Timeout: 30 * time.Second} } + apiClient := cloneHTTPClient(sharedClient) + apiClient.CheckRedirect = githubAPICheckRedirect(apiClient.CheckRedirect) // 下载客户端需要更长的超时时间 downloadClient, err := httpclient.GetClient(httpclient.Options{ @@ -56,13 +60,50 @@ func NewGitHubReleaseClient(proxyURL string, allowDirectOnProxyError bool) servi } downloadClient = &http.Client{Timeout: 10 * time.Minute} } + downloadClient = cloneHTTPClient(downloadClient) return &githubReleaseClient{ - httpClient: sharedClient, + httpClient: apiClient, downloadHTTPClient: downloadClient, + updateGitHubToken: os.Getenv("UPDATE_GITHUB_TOKEN"), } } +func cloneHTTPClient(client *http.Client) *http.Client { + cloned := *client + return &cloned +} + +func isGitHubAPIURL(url *url.URL) bool { + return url != nil && strings.EqualFold(url.Scheme, "https") && url.User == nil && + strings.EqualFold(url.Host, "api.github.com") +} + +func githubAPICheckRedirect(previous func(*http.Request, []*http.Request) error) func(*http.Request, []*http.Request) error { + return func(req *http.Request, via []*http.Request) error { + if !isGitHubAPIURL(req.URL) { + req.Header.Del("Authorization") + } + if previous != nil { + return previous(req, via) + } + return nil + } +} + +func (c *githubReleaseClient) newAPIRequest(ctx context.Context, url string) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/vnd.github.v3+json") + req.Header.Set("User-Agent", "Sub2API-Updater") + if c.updateGitHubToken != "" && isGitHubAPIURL(req.URL) { + req.Header.Set("Authorization", "Bearer "+c.updateGitHubToken) + } + return req, nil +} + func (c *githubReleaseClientError) FetchLatestRelease(ctx context.Context, repo string) (*service.GitHubRelease, error) { return nil, c.err } @@ -82,12 +123,10 @@ func (c *githubReleaseClientError) FetchChecksumFile(ctx context.Context, url st func (c *githubReleaseClient) FetchLatestRelease(ctx context.Context, repo string) (*service.GitHubRelease, error) { url := fmt.Sprintf("https://api.github.com/repos/%s/releases/latest", repo) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + req, err := c.newAPIRequest(ctx, url) if err != nil { return nil, err } - req.Header.Set("Accept", "application/vnd.github.v3+json") - req.Header.Set("User-Agent", "Sub2API-Updater") resp, err := c.httpClient.Do(req) if err != nil { @@ -116,12 +155,10 @@ func (c *githubReleaseClient) FetchRecentReleases(ctx context.Context, repo stri } url := fmt.Sprintf("https://api.github.com/repos/%s/releases?per_page=%d", repo, perPage) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + req, err := c.newAPIRequest(ctx, url) if err != nil { return nil, err } - req.Header.Set("Accept", "application/vnd.github.v3+json") - req.Header.Set("User-Agent", "Sub2API-Updater") resp, err := c.httpClient.Do(req) if err != nil { diff --git a/backend/internal/repository/github_release_service_test.go b/backend/internal/repository/github_release_service_test.go index abc9a604e6..3f93f0dfb6 100644 --- a/backend/internal/repository/github_release_service_test.go +++ b/backend/internal/repository/github_release_service_test.go @@ -3,6 +3,7 @@ package repository import ( "bytes" "context" + "io" "net/http" "net/http/httptest" "os" @@ -47,6 +48,97 @@ func newTestGitHubReleaseClient() *githubReleaseClient { } } +func TestGitHubReleaseClientAPIRequestAuthorization(t *testing.T) { + tests := []struct { + name string + url string + wantAuth string + }{ + {name: "exact HTTPS authority", url: "https://api.github.com/repos/test/repo", wantAuth: "Bearer update-secret"}, + {name: "HTTP", url: "http://api.github.com/repos/test/repo"}, + {name: "subdomain", url: "https://sub.api.github.com/repos/test/repo"}, + {name: "userinfo", url: "https://user@api.github.com/repos/test/repo"}, + {name: "explicit default port", url: "https://api.github.com:443/repos/test/repo"}, + {name: "custom port", url: "https://api.github.com:8443/repos/test/repo"}, + {name: "different host", url: "https://github.com/test/repo"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := newTestGitHubReleaseClient() + client.updateGitHubToken = "update-secret" + req, err := client.newAPIRequest(context.Background(), tt.url) + require.NoError(t, err) + require.Equal(t, tt.wantAuth, req.Header.Get("Authorization")) + }) + } + + client := newTestGitHubReleaseClient() + req, err := client.newAPIRequest(context.Background(), "https://api.github.com/repos/test/repo") + require.NoError(t, err) + require.Empty(t, req.Header.Get("Authorization")) +} + +func TestGitHubReleaseClientRedirectAuthorization(t *testing.T) { + tests := []struct { + name string + url string + wantAuth string + }{ + {name: "same HTTPS authority", url: "https://api.github.com/redirected", wantAuth: "Bearer update-secret"}, + {name: "HTTP", url: "http://api.github.com/redirected"}, + {name: "subdomain", url: "https://sub.api.github.com/redirected"}, + {name: "userinfo", url: "https://user@api.github.com/redirected"}, + {name: "custom port", url: "https://api.github.com:8443/redirected"}, + {name: "different host", url: "https://example.com/redirected"}, + } + + checkRedirect := githubAPICheckRedirect(nil) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, tt.url, nil) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer update-secret") + + require.NoError(t, checkRedirect(req, nil)) + require.Equal(t, tt.wantAuth, req.Header.Get("Authorization")) + }) + } +} + +func TestGitHubReleaseClientDoesNotAuthorizeDownloads(t *testing.T) { + client := newTestGitHubReleaseClient() + client.updateGitHubToken = "update-secret" + + var headers []http.Header + transport := githubReleaseRoundTripFunc(func(req *http.Request) (*http.Response, error) { + headers = append(headers, req.Header.Clone()) + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader("checksum")), + Request: req, + }, nil + }) + client.httpClient.Transport = transport + client.downloadHTTPClient.Transport = transport + + dest := filepath.Join(t.TempDir(), "asset") + require.NoError(t, client.DownloadFile(context.Background(), "https://objects.githubusercontent.com/asset", dest, 100)) + _, err := client.FetchChecksumFile(context.Background(), "https://github.com/test/repo/releases/download/v1/checksums.txt") + require.NoError(t, err) + require.Len(t, headers, 2) + for _, header := range headers { + require.Empty(t, header.Get("Authorization")) + } +} + +type githubReleaseRoundTripFunc func(*http.Request) (*http.Response, error) + +func (f githubReleaseRoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + func (s *GitHubReleaseServiceSuite) SetupTest() { s.tempDir = s.T().TempDir() } diff --git a/deploy/.env.example b/deploy/.env.example index 57056b4907..89b5ea0e94 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -23,6 +23,10 @@ SERVER_PORT=8080 # Server mode: release or debug SERVER_MODE=release +# Optional token used only for GitHub Release API update checks. Release asset +# downloads remain anonymous. GITHUB_TOKEN and GH_TOKEN are not used. +UPDATE_GITHUB_TOKEN= + # Return Server-Timing for authenticated requests made by the Admin web UI ENABLE_SERVER_TIMING=false diff --git a/deploy/README.md b/deploy/README.md index 84050b1883..e87f33fa25 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -238,6 +238,7 @@ docker compose down -v | `ADMIN_EMAIL` | No | `admin@sub2api.local` | Admin email | | `ADMIN_PASSWORD` | No | *(auto-generated)* | Admin password | | `TZ` | No | `Asia/Shanghai` | Timezone | +| `UPDATE_GITHUB_TOKEN` | No | *(empty)* | Token for `api.github.com` release checks only; asset downloads remain anonymous. | | `GEMINI_OAUTH_CLIENT_ID` | No | *(builtin)* | Google OAuth client ID (Gemini OAuth). Leave empty to use the built-in Gemini CLI client. | | `GEMINI_OAUTH_CLIENT_SECRET` | No | *(builtin)* | Google OAuth client secret (Gemini OAuth). Leave empty to use the built-in Gemini CLI client. | | `GEMINI_OAUTH_SCOPES` | No | *(default)* | OAuth scopes (Gemini OAuth) | diff --git a/deploy/docker-compose.local.yml b/deploy/docker-compose.local.yml index 5fb161603b..137d42c8df 100644 --- a/deploy/docker-compose.local.yml +++ b/deploy/docker-compose.local.yml @@ -53,6 +53,7 @@ services: - SERVER_MODE=${SERVER_MODE:-release} - ENABLE_SERVER_TIMING=${ENABLE_SERVER_TIMING:-false} - RUN_MODE=${RUN_MODE:-standard} + - UPDATE_GITHUB_TOKEN=${UPDATE_GITHUB_TOKEN:-} # ======================================================================= # Database Configuration (PostgreSQL) diff --git a/deploy/docker-compose.standalone.yml b/deploy/docker-compose.standalone.yml index 40ed4751d6..c5f0be7e6f 100644 --- a/deploy/docker-compose.standalone.yml +++ b/deploy/docker-compose.standalone.yml @@ -39,6 +39,7 @@ services: - SERVER_MODE=${SERVER_MODE:-release} - ENABLE_SERVER_TIMING=${ENABLE_SERVER_TIMING:-false} - RUN_MODE=${RUN_MODE:-standard} + - UPDATE_GITHUB_TOKEN=${UPDATE_GITHUB_TOKEN:-} # ======================================================================= # Database Configuration (PostgreSQL) - Required diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index e60776e580..7cca28a904 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -49,6 +49,7 @@ services: - SERVER_MODE=${SERVER_MODE:-release} - ENABLE_SERVER_TIMING=${ENABLE_SERVER_TIMING:-false} - RUN_MODE=${RUN_MODE:-standard} + - UPDATE_GITHUB_TOKEN=${UPDATE_GITHUB_TOKEN:-} # ======================================================================= # Database Configuration (PostgreSQL) diff --git a/deploy/install.sh b/deploy/install.sh index 1846dede35..246347c624 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -480,10 +480,58 @@ check_dependencies() { fi } +# Authenticate only GitHub REST API requests. Release asset downloads must stay anonymous. +github_api_curl() { + local arg + local expect_value=false + local url + + if [ "$#" -lt 1 ]; then + echo "github_api_curl requires exactly one GitHub API URL" >&2 + return 2 + fi + url="${!#}" + + # Keep authenticated invocations constrained to the options used below. In + # particular, curl config, --url, and --next could add another destination. + for arg in "${@:1:$#-1}"; do + if [ "$expect_value" = true ]; then + expect_value=false + continue + fi + case "$arg" in + -s|--silent) + ;; + --connect-timeout|--max-time|-o|--output|-w|--write-out) + expect_value=true + ;; + *) + echo "Unsafe github_api_curl argument: $arg" >&2 + return 2 + ;; + esac + done + + if [ "$expect_value" = true ] || [[ "$url" != https://api.github.com/* ]]; then + echo "github_api_curl requires exactly one GitHub API URL" >&2 + return 2 + fi + + if [ -n "${UPDATE_GITHUB_TOKEN:-}" ]; then + if [[ "$UPDATE_GITHUB_TOKEN" == *$'\n'* || "$UPDATE_GITHUB_TOKEN" == *$'\r'* || "$UPDATE_GITHUB_TOKEN" == *'"'* || "$UPDATE_GITHUB_TOKEN" == *'\'* ]]; then + echo "UPDATE_GITHUB_TOKEN contains unsupported characters" >&2 + return 2 + fi + printf 'header = "Authorization: Bearer %s"\n' "$UPDATE_GITHUB_TOKEN" | UPDATE_GITHUB_TOKEN= GITHUB_TOKEN= GH_TOKEN= curl -q --globoff --config - "$@" + else + UPDATE_GITHUB_TOKEN= GITHUB_TOKEN= GH_TOKEN= curl -q --globoff "$@" + fi +} + # Get latest release version get_latest_version() { print_info "$(msg 'fetching_version')" - LATEST_VERSION=$(curl -s --connect-timeout 10 --max-time 30 "https://api.github.com/repos/${GITHUB_REPO}/releases/latest" 2>/dev/null | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/') + LATEST_VERSION=$(github_api_curl -s --connect-timeout 10 --max-time 30 "https://api.github.com/repos/${GITHUB_REPO}/releases/latest" 2>/dev/null | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/') if [ -z "$LATEST_VERSION" ]; then print_error "$(msg 'failed_get_version')" @@ -499,7 +547,7 @@ list_versions() { print_info "$(msg 'fetching_versions')" local versions - versions=$(curl -s --connect-timeout 10 --max-time 30 "https://api.github.com/repos/${GITHUB_REPO}/releases" 2>/dev/null | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/' | head -20) + versions=$(github_api_curl -s --connect-timeout 10 --max-time 30 "https://api.github.com/repos/${GITHUB_REPO}/releases" 2>/dev/null | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/' | head -20) if [ -z "$versions" ]; then print_error "$(msg 'failed_get_version')" @@ -536,7 +584,7 @@ validate_version() { # Check if the release exists local http_code - http_code=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 10 --max-time 30 "https://api.github.com/repos/${GITHUB_REPO}/releases/tags/${version}" 2>/dev/null) + http_code=$(github_api_curl -s -o /dev/null -w "%{http_code}" --connect-timeout 10 --max-time 30 "https://api.github.com/repos/${GITHUB_REPO}/releases/tags/${version}" 2>/dev/null) # Check for network errors (empty or non-numeric response) if [ -z "$http_code" ] || ! [[ "$http_code" =~ ^[0-9]+$ ]]; then diff --git a/deploy/tests/install-github-token-test.sh b/deploy/tests/install-github-token-test.sh new file mode 100755 index 0000000000..4bfdb5feec --- /dev/null +++ b/deploy/tests/install-github-token-test.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +set -euo pipefail + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +TEMP_DIR=$(mktemp -d) +trap 'rm -rf "$TEMP_DIR"' EXIT + +cat > "$TEMP_DIR/curl" <<'EOF' +#!/bin/bash +printf '%s\n' "$@" > "$CURL_ARGS_LOG" +env > "${CURL_ARGS_LOG}.env" +cat > "${CURL_ARGS_LOG}.stdin" +EOF +chmod +x "$TEMP_DIR/curl" + +mkdir "$TEMP_DIR/home" +cat > "$TEMP_DIR/home/.curlrc" <<'EOF' +url = "https://example.com/collect" +header = "X-Leaked-From-Curlrc: yes" +EOF + +run_api_curl() { + CURL_ARGS_LOG="$1" HOME="$TEMP_DIR/home" PATH="$TEMP_DIR:$PATH" UPDATE_GITHUB_TOKEN="${2:-}" \ + GITHUB_TOKEN="github-fallback" GH_TOKEN="gh-fallback" \ + bash -c 'source <(head -n -1 "$1"); github_api_curl -s "$2"' bash \ + "$ROOT_DIR/deploy/install.sh" "https://api.github.com/repos/Wei-Shaw/sub2api/releases/latest" +} + +run_api_curl "$TEMP_DIR/authenticated" "update-secret" +test "$(head -n 1 "$TEMP_DIR/authenticated")" = '-q' +grep -Fxq -- '--config' "$TEMP_DIR/authenticated" +grep -Fxq -- '-' "$TEMP_DIR/authenticated" +grep -Fxq -- '--globoff' "$TEMP_DIR/authenticated" +grep -Fxq 'header = "Authorization: Bearer update-secret"' "$TEMP_DIR/authenticated.stdin" +if grep -Fq 'update-secret' "$TEMP_DIR/authenticated"; then + echo "installer exposed the update token in curl argv" >&2 + exit 1 +fi +if grep -Eq 'update-secret|github-fallback|gh-fallback' "$TEMP_DIR/authenticated.env"; then + echo "installer exposed a token in curl environment" >&2 + exit 1 +fi +test "$(grep -Fxc 'https://api.github.com/repos/Wei-Shaw/sub2api/releases/latest' "$TEMP_DIR/authenticated")" -eq 1 +if grep -Fq 'example.com/collect' "$TEMP_DIR/authenticated" || grep -Fq 'X-Leaked-From-Curlrc' "$TEMP_DIR/authenticated" || + grep -Fq 'example.com/collect' "$TEMP_DIR/authenticated.stdin" || grep -Fq 'X-Leaked-From-Curlrc' "$TEMP_DIR/authenticated.stdin"; then + echo "installer allowed hostile curl config into authenticated invocation" >&2 + exit 1 +fi + +run_api_curl "$TEMP_DIR/anonymous" +test "$(head -n 1 "$TEMP_DIR/anonymous")" = '-q' +if grep -Eq 'github-fallback|gh-fallback' "$TEMP_DIR/anonymous.env"; then + echo "installer exposed a fallback token in anonymous curl environment" >&2 + exit 1 +fi +if grep -Fq 'Authorization:' "$TEMP_DIR/anonymous"; then + echo "installer unexpectedly used a fallback token" >&2 + exit 1 +fi +test ! -s "$TEMP_DIR/anonymous.stdin" +test "$(grep -Fxc 'https://api.github.com/repos/Wei-Shaw/sub2api/releases/latest' "$TEMP_DIR/anonymous")" -eq 1 +if grep -Fq 'example.com/collect' "$TEMP_DIR/anonymous" || grep -Fq 'X-Leaked-From-Curlrc' "$TEMP_DIR/anonymous"; then + echo "installer allowed hostile curl config into anonymous invocation" >&2 + exit 1 +fi + +assert_unsafe_invocation_rejected() { + local name=$1 + shift + rm -f "$TEMP_DIR/$name" "$TEMP_DIR/$name.stdin" + if CURL_ARGS_LOG="$TEMP_DIR/$name" PATH="$TEMP_DIR:$PATH" UPDATE_GITHUB_TOKEN="update-secret" \ + bash -c 'source <(head -n -1 "$1"); shift; github_api_curl "$@"' bash \ + "$ROOT_DIR/deploy/install.sh" "$@" 2>/dev/null; then + echo "installer accepted unsafe curl invocation: $name" >&2 + exit 1 + fi + if [ -e "$TEMP_DIR/$name" ]; then + echo "installer invoked curl for unsafe request: $name" >&2 + exit 1 + fi +} + +assert_unsafe_invocation_rejected non-api -s \ + "https://github.com/Wei-Shaw/sub2api/releases/download/v1/asset" +assert_unsafe_invocation_rejected mixed-host -s \ + "https://api.github.com/repos/Wei-Shaw/sub2api/releases/latest" \ + "https://example.com/collect" +assert_unsafe_invocation_rejected multiple-api -s \ + "https://api.github.com/repos/Wei-Shaw/sub2api/releases/latest" \ + "https://api.github.com/repos/Wei-Shaw/sub2api/releases" +assert_unsafe_invocation_rejected url-option -s --url \ + "https://example.com/collect" \ + "https://api.github.com/repos/Wei-Shaw/sub2api/releases/latest" + +# Every installer release API request must use the scoped helper. +test "$(grep -c 'github_api_curl .*https://api.github.com/' "$ROOT_DIR/deploy/install.sh")" -eq 3 + +# Asset and checksum downloads must continue to call curl directly. +grep -Fq 'curl -sL "$download_url"' "$ROOT_DIR/deploy/install.sh" +grep -Fq 'curl -sL "$checksum_url"' "$ROOT_DIR/deploy/install.sh" + +echo "install GitHub token checks passed" From 361983cf4d3519717a4e9c869a0533d38a2a8180 Mon Sep 17 00:00:00 2001 From: Jlypx Date: Sun, 19 Jul 2026 11:23:26 +0800 Subject: [PATCH 03/59] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E8=BF=90?= =?UTF-8?q?=E7=BB=B4=E5=AE=9A=E6=97=B6=E6=8A=A5=E8=A1=A8=E9=82=AE=E4=BB=B6?= =?UTF-8?q?=E6=A8=A1=E6=9D=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/notification_email_service.go | 332 ++++++++++++++++-- .../notification_email_service_test.go | 120 ++++++- .../service/ops_scheduled_report_service.go | 205 +++++++++-- .../ops_scheduled_report_service_test.go | 143 ++++++++ .../admin/settings/EmailTemplateEditor.vue | 31 +- 5 files changed, 768 insertions(+), 63 deletions(-) create mode 100644 backend/internal/service/ops_scheduled_report_service_test.go diff --git a/backend/internal/service/notification_email_service.go b/backend/internal/service/notification_email_service.go index b7acccc7aa..76d8dbd7e6 100644 --- a/backend/internal/service/notification_email_service.go +++ b/backend/internal/service/notification_email_service.go @@ -51,6 +51,31 @@ var ( notificationEmailPlaceholderPattern = regexp.MustCompile(`{{\s*([a-zA-Z][a-zA-Z0-9_]*)\s*}}`) notificationEmailLocales = []string{notificationEmailDefaultLocale, notificationEmailLocaleChinese} notificationEmailCommonPlaceholders = []string{"site_name", "recipient_name", "recipient_email"} + // Keep summary values separate so admins can rearrange or omit individual metrics in the template. + notificationEmailOpsSummaryPlaceholders = []string{ + "report_summary_display", + "report_total_requests", + "report_success_count", + "report_sla_error_count", + "report_business_limited_count", + "report_sla", + "report_error_rate", + "report_upstream_error_rate", + "report_upstream_error_count_excl_429_529", + "report_upstream_429_count", + "report_upstream_529_count", + "report_latency_p50", + "report_latency_p99", + "report_ttft_p50", + "report_ttft_p99", + "report_tokens", + "report_qps_current", + "report_qps_peak", + "report_qps_avg", + "report_tps_current", + "report_tps_peak", + "report_tps_avg", + } ) type NotificationEmailService struct { @@ -504,6 +529,34 @@ func (s *NotificationEmailService) runtimeVariables(ctx context.Context, event, for key, value := range input.Variables { variables[key] = value } + if event == NotificationEmailEventOpsScheduledReport { + // Scheduled reports may be sent by integrations that only provide report_html. + // Do not let preview sample values appear in a live email in that case. + if _, ok := input.Variables["report_html"]; !ok { + variables["report_html"] = "" + } + if _, ok := input.Variables["report_detail_display"]; !ok { + // Keep legacy/custom templates useful when they only render report_html. + variables["report_detail_display"] = "block" + } + hasSummaryValues := false + for _, placeholder := range notificationEmailOpsSummaryPlaceholders { + if _, ok := input.Variables[placeholder]; ok { + if placeholder != "report_summary_display" { + hasSummaryValues = true + } + continue + } + variables[placeholder] = "-" + } + if _, ok := input.Variables["report_summary_display"]; !ok { + if hasSummaryValues { + variables["report_summary_display"] = "block" + } else { + variables["report_summary_display"] = "none" + } + } + } variables["site_name"] = s.siteName(ctx) variables["recipient_email"] = input.RecipientEmail if strings.TrimSpace(input.RecipientName) != "" { @@ -844,7 +897,7 @@ func isSafeNotificationEmailURL(raw string) bool { func notificationEmailSampleVariables(locale string) map[string]string { if normalizeNotificationLocale(locale) == notificationEmailLocaleChinese { - return map[string]string{ + variables := map[string]string{ "site_name": defaultSiteName, "recipient_name": "张三", "recipient_email": "user@example.com", @@ -885,12 +938,14 @@ func notificationEmailSampleVariables(locale string) map[string]string { "alert_description": "最近 10 分钟错误率超过阈值", "report_name": "日报", "report_type": "daily_summary", - "report_start_time": "2026-05-19 12:00", - "report_end_time": "2026-05-20 12:00", - "report_html": "

日报

请求量：1024

", + "report_start_time": "2026-07-18T01:00:26Z", + "report_end_time": "2026-07-19T01:00:26Z", + "report_html": "

日报

请求量：2,374

", } + addNotificationEmailOpsSummarySampleVariables(variables) + return variables } - return map[string]string{ + variables := map[string]string{ "site_name": defaultSiteName, "recipient_name": "Alex", "recipient_email": "user@example.com", @@ -931,10 +986,38 @@ func notificationEmailSampleVariables(locale string) map[string]string { "alert_description": "Error rate exceeded threshold in the last 10 minutes.", "report_name": "Daily summary", "report_type": "daily_summary", - "report_start_time": "2026-05-19 12:00", - "report_end_time": "2026-05-20 12:00", - "report_html": "

Daily summary

Requests: 1024

", + "report_start_time": "2026-07-18T01:00:26Z", + "report_end_time": "2026-07-19T01:00:26Z", + "report_html": "

Daily summary

Requests: 2,374

", } + addNotificationEmailOpsSummarySampleVariables(variables) + return variables +} + +func addNotificationEmailOpsSummarySampleVariables(variables map[string]string) { + variables["report_summary_display"] = "block" + variables["report_detail_display"] = "none" + variables["report_total_requests"] = "2,374" + variables["report_success_count"] = "1,451" + variables["report_sla_error_count"] = "2" + variables["report_business_limited_count"] = "921" + variables["report_sla"] = "99.86%" + variables["report_error_rate"] = "0.14%" + variables["report_upstream_error_rate"] = "0.28%" + variables["report_upstream_error_count_excl_429_529"] = "4" + variables["report_upstream_429_count"] = "0" + variables["report_upstream_529_count"] = "0" + variables["report_latency_p50"] = "8,231 ms" + variables["report_latency_p99"] = "151,260 ms" + variables["report_ttft_p50"] = "1,674 ms" + variables["report_ttft_p99"] = "11,222 ms" + variables["report_tokens"] = "121,550,190" + variables["report_qps_current"] = "0.0" + variables["report_qps_peak"] = "1.2" + variables["report_qps_avg"] = "0.0" + variables["report_tps_current"] = "0.0" + variables["report_tps_peak"] = "133421.2" + variables["report_tps_avg"] = "1406.8" } var notificationEmailEventOrder = []string{ @@ -1061,8 +1144,13 @@ var notificationEmailEventDefinitions = map[string]NotificationEmailEventInfo{ Description: "Sent to configured operations recipients for scheduled daily/weekly/error/account-health reports.", Category: "ops", Optional: false, - Placeholders: append(append([]string{}, notificationEmailCommonPlaceholders...), - "report_name", "report_type", "report_start_time", "report_end_time", "report_html"), + Placeholders: append( + append( + append([]string{}, notificationEmailCommonPlaceholders...), + "report_name", "report_type", "report_start_time", "report_end_time", + ), + append(append([]string{}, notificationEmailOpsSummaryPlaceholders...), "report_detail_display", "report_html")..., + ), }, } @@ -1294,11 +1382,11 @@ var notificationEmailOfficialTemplates = map[string]map[string]notificationEmail HTML: notificationEmailCard("#ef4444", "Cyber-security policy notice", `

Hello {{recipient_name}},

Your request was blocked by the upstream provider's cyber-security policy.

- - - - - +
Triggered at{{triggered_at}}
Model{{model}}
Group{{group_name}}
Upstream message{{upstream_message}}
+ + + +
Triggered at{{triggered_at}}
Model{{model}}
Group{{group_name}}
Upstream message{{upstream_message}}

If you believe this is a mistake, try rephrasing your request, or apply for authorized security access.

`), }, @@ -1307,11 +1395,11 @@ var notificationEmailOfficialTemplates = map[string]map[string]notificationEmail HTML: notificationEmailCard("#ef4444", "网络安全策略拦截提醒", `

{{recipient_name}}，您好：

您的请求被上游服务商的网络安全策略（cyber policy）拦截。

- - - - - +
触发时间{{triggered_at}}
模型{{model}}
所属分组{{group_name}}
上游说明{{upstream_message}}
+ + + +
触发时间{{triggered_at}}
模型{{model}}
所属分组{{group_name}}
上游说明{{upstream_message}}

如认为系误判，可调整请求措辞后重试，或申请获得授权的安全访问权限。

`), }, @@ -1341,23 +1429,207 @@ var notificationEmailOfficialTemplates = map[string]map[string]notificationEmail NotificationEmailEventOpsScheduledReport: { notificationEmailDefaultLocale: { Subject: "[Ops Report] {{report_name}}", - HTML: notificationEmailCard("#0891b2", "Ops report", ` -

Report: {{report_name}}

-

Type: {{report_type}}

-

Range: {{report_start_time}} - {{report_end_time}}

-
{{report_html}}
`), + HTML: notificationEmailOpsScheduledReportTemplate(notificationEmailDefaultLocale), }, notificationEmailLocaleChinese: { Subject: "[运维报表] {{report_name}}", - HTML: notificationEmailCard("#0891b2", "运维报表", ` -

报表：{{report_name}}

-

类型：{{report_type}}

-

时间范围：{{report_start_time}} - {{report_end_time}}

-
{{report_html}}
`), + HTML: notificationEmailOpsScheduledReportTemplate(notificationEmailLocaleChinese), }, }, } +func notificationEmailOpsScheduledReportTemplate(locale string) string { + if normalizeNotificationLocale(locale) == notificationEmailLocaleChinese { + return ` + + + + + + + +
+
+

运维报表

+

{{report_name}}

+

{{site_name}} 的运行概览

+
+
+ + + + + + +
+

请求概览

+ + + + + + + + +

可靠性

+ + + + + + + + +

延迟表现

+ + + + + +

吞吐量

+ + + + + + +
+
{{report_html}}
+
+
此邮件由 {{site_name}} 自动发送，请勿直接回复。
+
+ +` + } + + return ` + + + + + + + +
+
+

Operations report

+

{{report_name}}

+

{{site_name}} runtime overview

+
+
+ + + + + + +
+

Request Overview

+ + + + + + + + +

Reliability

+ + + + + + + + +

Latency

+ + + + + +

Throughput

+ + + + + + +
+
{{report_html}}
+
+
This email was sent automatically by {{site_name}}. Please do not reply directly.
+
+ +` +} + func notificationEmailCard(accent, title, content string) string { return ` diff --git a/backend/internal/service/notification_email_service_test.go b/backend/internal/service/notification_email_service_test.go index 9faeda7993..975436670f 100644 --- a/backend/internal/service/notification_email_service_test.go +++ b/backend/internal/service/notification_email_service_test.go @@ -158,6 +158,104 @@ func TestNotificationEmailAdditionalEventsAreListedAndPreviewable(t *testing.T) } } +func TestCyberPolicyNoticeTemplateWrapsLongUpstreamMessages(t *testing.T) { + ctx := context.Background() + svc := NewNotificationEmailService(newNotificationEmailMemorySettingRepo(), nil) + longMessage := strings.Repeat("0123456789abcdef", 256) + + for _, locale := range []string{"en", "zh"} { + preview, err := svc.PreviewTemplate(ctx, NotificationEmailPreviewInput{ + Event: NotificationEmailEventCyberPolicyNotice, + Locale: locale, + Variables: map[string]string{ + "upstream_message": longMessage, + }, + }) + require.NoError(t, err) + require.Contains(t, preview.HTML, "table-layout:fixed") + require.Contains(t, preview.HTML, "overflow-wrap:anywhere") + require.Contains(t, preview.HTML, "word-break:break-all") + require.NotContains(t, preview.HTML, "{{upstream_message}}") + } +} + +func TestOpsScheduledReportTemplateExposesEditableSummaryMetrics(t *testing.T) { + ctx := context.Background() + svc := NewNotificationEmailService(newNotificationEmailMemorySettingRepo(), nil) + + requiredPlaceholders := []string{ + "report_summary_display", + "report_detail_display", + "report_total_requests", + "report_success_count", + "report_sla_error_count", + "report_business_limited_count", + "report_sla", + "report_error_rate", + "report_upstream_error_rate", + "report_upstream_error_count_excl_429_529", + "report_upstream_429_count", + "report_upstream_529_count", + "report_latency_p50", + "report_latency_p99", + "report_ttft_p50", + "report_ttft_p99", + "report_tokens", + "report_qps_current", + "report_qps_peak", + "report_qps_avg", + "report_tps_current", + "report_tps_peak", + "report_tps_avg", + } + + for _, locale := range []string{"en", "zh"} { + tmpl, err := svc.GetTemplate(ctx, NotificationEmailEventOpsScheduledReport, locale) + require.NoError(t, err) + for _, placeholder := range requiredPlaceholders { + require.Contains(t, tmpl.Placeholders, placeholder) + require.Contains(t, tmpl.HTML, "{{"+placeholder+"}}") + } + + preview, err := svc.PreviewTemplate(ctx, NotificationEmailPreviewInput{ + Event: NotificationEmailEventOpsScheduledReport, + Locale: locale, + }) + require.NoError(t, err) + require.Contains(t, preview.HTML, "2,374") + require.Contains(t, preview.HTML, "99.86%") + require.Contains(t, preview.HTML, "151,260 ms") + require.Contains(t, preview.HTML, `style="display: none;"`) + require.NotContains(t, preview.HTML, "{{report_total_requests}}") + } +} + +func TestOpsScheduledReportRuntimeVariablesDoNotLeakPreviewSamples(t *testing.T) { + ctx := context.Background() + svc := NewNotificationEmailService(newNotificationEmailMemorySettingRepo(), nil) + + variables := svc.runtimeVariables(ctx, NotificationEmailEventOpsScheduledReport, "en", NotificationEmailSendInput{}) + require.Equal(t, "none", variables["report_summary_display"]) + require.Equal(t, "block", variables["report_detail_display"]) + require.Empty(t, variables["report_html"]) + for _, placeholder := range notificationEmailOpsSummaryPlaceholders { + if placeholder == "report_summary_display" { + continue + } + require.Equal(t, "-", variables[placeholder]) + } + + rendered, err := renderNotificationEmail( + NotificationEmailEventOpsScheduledReport, + "Report", + `
{{report_html}}
`, + variables, + nil, + ) + require.NoError(t, err) + require.NotContains(t, rendered.HTML, "

Daily summary

") +} + func TestNotificationEmailRawHTMLVariablesAreTrustedOnlyForHTMLPlaceholders(t *testing.T) { require.True(t, notificationEmailRawHTMLAllowed(NotificationEmailEventOpsScheduledReport, "report_html")) require.False(t, notificationEmailRawHTMLAllowed(NotificationEmailEventOpsScheduledReport, "recipient_name")) @@ -455,9 +553,11 @@ func TestNotificationEmailMemorySettingRepoSatisfiesInterface(t *testing.T) { } type notificationEmailTestSMTPServer struct { - listener net.Listener - wg sync.WaitGroup - messages atomic.Int64 + listener net.Listener + wg sync.WaitGroup + messages atomic.Int64 + messageMu sync.Mutex + messageBodies []string } func startNotificationEmailTestSMTPServer(t *testing.T) *notificationEmailTestSMTPServer { @@ -489,6 +589,15 @@ func (s *notificationEmailTestSMTPServer) messageCount() int64 { return s.messages.Load() } +func (s *notificationEmailTestSMTPServer) lastMessage() string { + s.messageMu.Lock() + defer s.messageMu.Unlock() + if len(s.messageBodies) == 0 { + return "" + } + return s.messageBodies[len(s.messageBodies)-1] +} + func (s *notificationEmailTestSMTPServer) close() { _ = s.listener.Close() s.wg.Wait() @@ -547,6 +656,7 @@ func (s *notificationEmailTestSMTPServer) handleConn(conn net.Conn) { if !writeLine("354 End data with .") { return } + var message strings.Builder for { dataLine, err := rw.ReadString('\n') if err != nil { @@ -555,7 +665,11 @@ func (s *notificationEmailTestSMTPServer) handleConn(conn net.Conn) { if strings.TrimRight(dataLine, "\r\n") == "." { break } + message.WriteString(dataLine) } + s.messageMu.Lock() + s.messageBodies = append(s.messageBodies, message.String()) + s.messageMu.Unlock() s.messages.Add(1) if !writeLine("250 2.0.0 OK") { return diff --git a/backend/internal/service/ops_scheduled_report_service.go b/backend/internal/service/ops_scheduled_report_service.go index 54aad11438..2ddb20c1db 100644 --- a/backend/internal/service/ops_scheduled_report_service.go +++ b/backend/internal/service/ops_scheduled_report_service.go @@ -219,6 +219,11 @@ type opsScheduledReport struct { NextRunAt time.Time } +type opsScheduledReportContent struct { + html string + overview *OpsDashboardOverview +} + func (s *OpsScheduledReportService) listScheduledReports(ctx context.Context, now time.Time) []*opsScheduledReport { if s == nil || s.opsService == nil { return nil @@ -316,11 +321,11 @@ func (s *OpsScheduledReportService) runReport(ctx context.Context, report *opsSc // Mark as "run" up-front so a broken SMTP config doesn't spam retries every minute. s.setLastRunAt(ctx, report.ReportType, now) - content, err := s.generateReportHTML(ctx, report, now) + content, err := s.generateReportContent(ctx, report, now) if err != nil { return 0, err } - if strings.TrimSpace(content) == "" { + if strings.TrimSpace(content.html) == "" { // Skip sending when the report decides not to emit content (e.g., digest below min count). return 0, nil } @@ -336,9 +341,6 @@ func (s *OpsScheduledReportService) runReport(ctx context.Context, report *opsSc return 0, nil } - subject := fmt.Sprintf("[Ops Report] %s", strings.TrimSpace(report.Name)) - templateVariables := opsScheduledReportEmailVariables(report, now) - attempts := 0 for _, to := range recipients { addr := strings.TrimSpace(to) @@ -346,25 +348,40 @@ func (s *OpsScheduledReportService) runReport(ctx context.Context, report *opsSc continue } attempts++ + locale := "" if s.emailService.notificationEmailService != nil { + locale = s.emailService.notificationEmailService.ResolveRecipientLocale(ctx, 0, addr) + templateVariables := opsScheduledReportLocalizedEmailVariables(report, now, locale) + rawHTMLVariables := map[string]string{"report_html": content.html} + if isOpsSummaryReport(report) { + templateVariables = opsSummaryReportEmailVariables(report, now, content.overview, locale) + } if err := s.emailService.notificationEmailService.Send(ctx, NotificationEmailSendInput{ - Event: NotificationEmailEventOpsScheduledReport, - RecipientEmail: addr, - RecipientName: emailRecipientName(addr), - SourceType: "ops_scheduled_report", - SourceID: opsScheduledReportDeliverySourceID(report), - ReminderKey: now.UTC().Format("2006-01-02T15:04"), - Variables: templateVariables, - RawHTMLVariables: map[string]string{ - "report_html": content, - }, + Event: NotificationEmailEventOpsScheduledReport, + Locale: locale, + RecipientEmail: addr, + RecipientName: emailRecipientName(addr), + SourceType: "ops_scheduled_report", + SourceID: opsScheduledReportDeliverySourceID(report), + ReminderKey: now.UTC().Format("2006-01-02T15:04"), + Variables: templateVariables, + RawHTMLVariables: rawHTMLVariables, }); err == nil { continue } else if !shouldFallbackNotificationEmail(err) { continue } } - if err := s.emailService.SendEmail(ctx, addr, subject, content); err != nil { + subjectName := strings.TrimSpace(report.Name) + if locale != "" { + subjectName = opsScheduledReportLocalizedName(report, locale) + } + subjectPrefix := "[Ops Report]" + if strings.HasPrefix(strings.ToLower(locale), "zh") { + subjectPrefix = "[运维报表]" + } + subject := fmt.Sprintf("%s %s", subjectPrefix, subjectName) + if err := s.emailService.SendEmail(ctx, addr, subject, content.html); err != nil { // Ignore per-recipient failures; continue best-effort. continue } @@ -412,12 +429,138 @@ func opsScheduledReportEmailVariables(report *opsScheduledReport, now time.Time) } } -func (s *OpsScheduledReportService) generateReportHTML(ctx context.Context, report *opsScheduledReport, now time.Time) (string, error) { +func opsScheduledReportLocalizedEmailVariables(report *opsScheduledReport, now time.Time, locale string) map[string]string { + variables := opsScheduledReportEmailVariables(report, now) + variables["report_html"] = "" + variables["report_detail_display"] = "block" + for _, placeholder := range notificationEmailOpsSummaryPlaceholders { + variables[placeholder] = "-" + } + variables["report_summary_display"] = "none" + if name := opsScheduledReportLocalizedName(report, locale); name != "" { + variables["report_name"] = name + } + return variables +} + +func opsScheduledReportLocalizedName(report *opsScheduledReport, locale string) string { + if report == nil { + return "Ops report" + } + chinese := strings.HasPrefix(strings.ToLower(strings.TrimSpace(locale)), "zh") + switch strings.TrimSpace(report.ReportType) { + case "daily_summary": + if chinese { + return "日报" + } + return "Daily summary" + case "weekly_summary": + if chinese { + return "周报" + } + return "Weekly summary" + case "error_digest": + if chinese { + return "错误摘要" + } + return "Error digest" + case "account_health": + if chinese { + return "账号健康" + } + return "Account health" + default: + return strings.TrimSpace(report.Name) + } +} + +func isOpsSummaryReport(report *opsScheduledReport) bool { + if report == nil { + return false + } + switch strings.TrimSpace(report.ReportType) { + case "daily_summary", "weekly_summary": + return true + default: + return false + } +} + +func opsSummaryReportEmailVariables(report *opsScheduledReport, now time.Time, overview *OpsDashboardOverview, locale string) map[string]string { + variables := opsScheduledReportLocalizedEmailVariables(report, now, locale) + variables["report_detail_display"] = "none" + if overview == nil { + for _, placeholder := range notificationEmailOpsSummaryPlaceholders { + if placeholder == "report_summary_display" { + continue + } + variables[placeholder] = "-" + } + variables["report_summary_display"] = "block" + return variables + } + variables["report_summary_display"] = "block" + + variables["report_total_requests"] = formatOpsReportInteger(overview.RequestCountTotal) + variables["report_success_count"] = formatOpsReportInteger(overview.SuccessCount) + variables["report_sla_error_count"] = formatOpsReportInteger(overview.ErrorCountSLA) + variables["report_business_limited_count"] = formatOpsReportInteger(overview.BusinessLimitedCount) + variables["report_sla"] = fmt.Sprintf("%.2f%%", overview.SLA*100) + variables["report_error_rate"] = fmt.Sprintf("%.2f%%", overview.ErrorRate*100) + variables["report_upstream_error_rate"] = fmt.Sprintf("%.2f%%", overview.UpstreamErrorRate*100) + variables["report_upstream_error_count_excl_429_529"] = formatOpsReportInteger(overview.UpstreamErrorCountExcl429529) + variables["report_upstream_429_count"] = formatOpsReportInteger(overview.Upstream429Count) + variables["report_upstream_529_count"] = formatOpsReportInteger(overview.Upstream529Count) + variables["report_latency_p50"] = formatOpsReportMilliseconds(overview.Duration.P50) + variables["report_latency_p99"] = formatOpsReportMilliseconds(overview.Duration.P99) + variables["report_ttft_p50"] = formatOpsReportMilliseconds(overview.TTFT.P50) + variables["report_ttft_p99"] = formatOpsReportMilliseconds(overview.TTFT.P99) + variables["report_tokens"] = formatOpsReportInteger(overview.TokenConsumed) + variables["report_qps_current"] = fmt.Sprintf("%.1f", overview.QPS.Current) + variables["report_qps_peak"] = fmt.Sprintf("%.1f", overview.QPS.Peak) + variables["report_qps_avg"] = fmt.Sprintf("%.1f", overview.QPS.Avg) + variables["report_tps_current"] = fmt.Sprintf("%.1f", overview.TPS.Current) + variables["report_tps_peak"] = fmt.Sprintf("%.1f", overview.TPS.Peak) + variables["report_tps_avg"] = fmt.Sprintf("%.1f", overview.TPS.Avg) + return variables +} + +func formatOpsReportInteger(value int64) string { + raw := strconv.FormatInt(value, 10) + start := 0 + if strings.HasPrefix(raw, "-") { + start = 1 + } + if len(raw)-start <= 3 { + return raw + } + + var builder strings.Builder + builder.Grow(len(raw) + (len(raw)-start-1)/3) + builder.WriteString(raw[:start]) + digitLen := len(raw) - start + for offset := 0; offset < digitLen; offset++ { + if offset > 0 && (digitLen-offset)%3 == 0 { + builder.WriteByte(',') + } + builder.WriteByte(raw[start+offset]) + } + return builder.String() +} + +func formatOpsReportMilliseconds(value *int) string { + if value == nil { + return "-" + } + return fmt.Sprintf("%s ms", formatOpsReportInteger(int64(*value))) +} + +func (s *OpsScheduledReportService) generateReportContent(ctx context.Context, report *opsScheduledReport, now time.Time) (opsScheduledReportContent, error) { if s == nil || s.opsService == nil || report == nil { - return "", fmt.Errorf("service not initialized") + return opsScheduledReportContent{}, fmt.Errorf("service not initialized") } if report.TimeRange <= 0 { - return "", fmt.Errorf("invalid time range") + return opsScheduledReportContent{}, fmt.Errorf("invalid time range") } end := now.UTC() @@ -444,10 +587,13 @@ func (s *OpsScheduledReportService) generateReportHTML(ctx context.Context, repo }) } if err != nil { - return "", err + return opsScheduledReportContent{}, err } } - return buildOpsSummaryEmailHTML(report.Name, start, end, overview), nil + return opsScheduledReportContent{ + html: buildOpsSummaryEmailHTML(report.Name, start, end, overview), + overview: overview, + }, nil case "error_digest": // Lightweight digest: list recent errors (status>=400) and breakdown by type. startTime := start @@ -460,25 +606,30 @@ func (s *OpsScheduledReportService) generateReportHTML(ctx context.Context, repo } out, err := s.opsService.GetErrorLogs(ctx, filter) if err != nil { - return "", err + return opsScheduledReportContent{}, err } if report.ErrorDigestMinCount > 0 && out != nil && out.Total < report.ErrorDigestMinCount { - return "", nil + return opsScheduledReportContent{}, nil } - return buildOpsErrorDigestEmailHTML(report.Name, start, end, out), nil + return opsScheduledReportContent{html: buildOpsErrorDigestEmailHTML(report.Name, start, end, out)}, nil case "account_health": // Best-effort: use account availability (not error rate yet). avail, err := s.opsService.GetAccountAvailability(ctx, "", nil) if err != nil { - return "", err + return opsScheduledReportContent{}, err } _ = report.AccountHealthErrorRateThreshold // reserved for future per-account error rate report - return buildOpsAccountHealthEmailHTML(report.Name, start, end, avail), nil + return opsScheduledReportContent{html: buildOpsAccountHealthEmailHTML(report.Name, start, end, avail)}, nil default: - return "", fmt.Errorf("unknown report type: %s", report.ReportType) + return opsScheduledReportContent{}, fmt.Errorf("unknown report type: %s", report.ReportType) } } +func (s *OpsScheduledReportService) generateReportHTML(ctx context.Context, report *opsScheduledReport, now time.Time) (string, error) { + content, err := s.generateReportContent(ctx, report, now) + return content.html, err +} + func buildOpsSummaryEmailHTML(title string, start, end time.Time, overview *OpsDashboardOverview) string { if overview == nil { return fmt.Sprintf("

%s

No data.

", htmlEscape(title)) diff --git a/backend/internal/service/ops_scheduled_report_service_test.go b/backend/internal/service/ops_scheduled_report_service_test.go new file mode 100644 index 0000000000..e4bfcbf05b --- /dev/null +++ b/backend/internal/service/ops_scheduled_report_service_test.go @@ -0,0 +1,143 @@ +package service + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestOpsSummaryReportEmailVariables(t *testing.T) { + latencyP50 := 8231 + latencyP99 := 151260 + ttftP50 := 1674 + ttftP99 := 11222 + now := time.Date(2026, time.July, 19, 1, 0, 26, 0, time.UTC) + report := &opsScheduledReport{ + Name: "日报", + ReportType: "daily_summary", + TimeRange: 24 * time.Hour, + } + overview := &OpsDashboardOverview{ + RequestCountTotal: 2374, + SuccessCount: 1451, + ErrorCountSLA: 2, + BusinessLimitedCount: 921, + SLA: 0.9986, + ErrorRate: 0.0014, + UpstreamErrorRate: 0.0028, + UpstreamErrorCountExcl429529: 4, + Upstream429Count: 0, + Upstream529Count: 0, + TokenConsumed: 121550190, + Duration: OpsPercentiles{P50: &latencyP50, P99: &latencyP99}, + TTFT: OpsPercentiles{P50: &ttftP50, P99: &ttftP99}, + QPS: OpsRateSummary{Current: 0, Peak: 1.2, Avg: 0}, + TPS: OpsRateSummary{Current: 0, Peak: 133421.2, Avg: 1406.8}, + } + + variables := opsSummaryReportEmailVariables(report, now, overview, "en") + require.Equal(t, "Daily summary", variables["report_name"]) + require.Equal(t, "daily_summary", variables["report_type"]) + require.Equal(t, "2026-07-18T01:00:26Z", variables["report_start_time"]) + require.Equal(t, "2026-07-19T01:00:26Z", variables["report_end_time"]) + require.Equal(t, "2,374", variables["report_total_requests"]) + require.Equal(t, "1,451", variables["report_success_count"]) + require.Equal(t, "2", variables["report_sla_error_count"]) + require.Equal(t, "921", variables["report_business_limited_count"]) + require.Equal(t, "99.86%", variables["report_sla"]) + require.Equal(t, "0.14%", variables["report_error_rate"]) + require.Equal(t, "0.28%", variables["report_upstream_error_rate"]) + require.Equal(t, "4", variables["report_upstream_error_count_excl_429_529"]) + require.Equal(t, "8,231 ms", variables["report_latency_p50"]) + require.Equal(t, "151,260 ms", variables["report_latency_p99"]) + require.Equal(t, "1,674 ms", variables["report_ttft_p50"]) + require.Equal(t, "11,222 ms", variables["report_ttft_p99"]) + require.Equal(t, "121,550,190", variables["report_tokens"]) + require.Equal(t, "1.2", variables["report_qps_peak"]) + require.Equal(t, "133421.2", variables["report_tps_peak"]) + require.Equal(t, "1406.8", variables["report_tps_avg"]) + require.Empty(t, variables["report_html"]) + require.Equal(t, "none", variables["report_detail_display"]) + + zhVariables := opsSummaryReportEmailVariables(report, now, overview, "zh-CN") + require.Equal(t, "日报", zhVariables["report_name"]) + emptyVariables := opsSummaryReportEmailVariables(report, now, nil, "en") + require.Equal(t, "block", emptyVariables["report_summary_display"]) + require.Equal(t, "none", emptyVariables["report_detail_display"]) + + legacyTemplate, err := renderNotificationEmail( + NotificationEmailEventOpsScheduledReport, + "Report", + `
{{report_html}}
`, + variables, + map[string]string{"report_html": `

generated summary

`}, + ) + require.NoError(t, err) + require.Contains(t, legacyTemplate.HTML, `

generated summary

`) +} + +func TestOpsScheduledReportVariablesDoNotUsePreviewMetrics(t *testing.T) { + now := time.Date(2026, time.July, 19, 1, 0, 26, 0, time.UTC) + report := &opsScheduledReport{ + Name: "错误摘要", + ReportType: "error_digest", + TimeRange: 24 * time.Hour, + } + + variables := opsScheduledReportLocalizedEmailVariables(report, now, "en") + require.Equal(t, "Error digest", variables["report_name"]) + require.Empty(t, variables["report_html"]) + require.Equal(t, "block", variables["report_detail_display"]) + for _, placeholder := range notificationEmailOpsSummaryPlaceholders { + if placeholder == "report_summary_display" { + require.Equal(t, "none", variables[placeholder]) + continue + } + require.Equal(t, "-", variables[placeholder]) + } +} + +func TestOpsScheduledReportLegacyTemplateReceivesSummaryHTML(t *testing.T) { + ctx := context.Background() + repo := newNotificationEmailMemorySettingRepo() + smtpServer := startNotificationEmailTestSMTPServer(t) + require.NoError(t, repo.SetMultiple(ctx, smtpServer.settings())) + + emailService := NewEmailService(repo, nil) + notificationService := NewNotificationEmailService(repo, emailService) + _, err := notificationService.UpdateTemplate( + ctx, + NotificationEmailEventOpsScheduledReport, + "en", + "Legacy report {{report_name}}", + `
{{report_html}}
`, + ) + require.NoError(t, err) + + svc := &OpsScheduledReportService{ + opsService: &OpsService{opsRepo: &opsRepoMock{}}, + emailService: emailService, + } + report := &opsScheduledReport{ + Name: "日报", + ReportType: "daily_summary", + TimeRange: 24 * time.Hour, + Recipients: []string{"ops@example.com"}, + } + + attempts, err := svc.runReport(ctx, report, time.Date(2026, time.July, 19, 1, 0, 26, 0, time.UTC)) + require.NoError(t, err) + require.Equal(t, 1, attempts) + require.Equal(t, int64(1), smtpServer.messageCount()) + message := smtpServer.lastMessage() + require.Contains(t, message, `
`) + require.Contains(t, message, `

日报

`) +} + +func TestFormatOpsReportIntegerGroupsDigits(t *testing.T) { + require.Equal(t, "2,374", formatOpsReportInteger(2374)) + require.Equal(t, "-1,234", formatOpsReportInteger(-1234)) + require.Equal(t, "42", formatOpsReportInteger(42)) +} diff --git a/frontend/src/views/admin/settings/EmailTemplateEditor.vue b/frontend/src/views/admin/settings/EmailTemplateEditor.vue index 5fcb371e6c..3e28ede484 100644 --- a/frontend/src/views/admin/settings/EmailTemplateEditor.vue +++ b/frontend/src/views/admin/settings/EmailTemplateEditor.vue @@ -286,6 +286,29 @@ const fallbackPlaceholders = [ "{{report_type}}", "{{report_start_time}}", "{{report_end_time}}", + "{{report_summary_display}}", + "{{report_detail_display}}", + "{{report_total_requests}}", + "{{report_success_count}}", + "{{report_sla_error_count}}", + "{{report_business_limited_count}}", + "{{report_sla}}", + "{{report_error_rate}}", + "{{report_upstream_error_rate}}", + "{{report_upstream_error_count_excl_429_529}}", + "{{report_upstream_429_count}}", + "{{report_upstream_529_count}}", + "{{report_latency_p50}}", + "{{report_latency_p99}}", + "{{report_ttft_p50}}", + "{{report_ttft_p99}}", + "{{report_tokens}}", + "{{report_qps_current}}", + "{{report_qps_peak}}", + "{{report_qps_avg}}", + "{{report_tps_current}}", + "{{report_tps_peak}}", + "{{report_tps_avg}}", "{{report_html}}", ]; @@ -374,7 +397,7 @@ const eventDisplayMeta: Record = { }, "ops.scheduled_report": { label: "运维定时报表", - timing: "运维日报、周报、错误摘要或账号健康报表到达配置的发送时间时发送。", + timing: "运维日报、周报、错误摘要或账号健康报表到达配置的发送时间时发送；日报和周报的完整指标均可在模板中编辑。", categoryLabel: "运维", }, }; @@ -437,7 +460,7 @@ const eventDisplayMetaEn: Record = { }, "ops.scheduled_report": { label: "Ops Scheduled Report", - timing: "Sent when a configured daily, weekly, error digest, or account health report reaches its scheduled send time.", + timing: "Sent when a configured daily, weekly, error digest, or account health report reaches its scheduled send time. Every daily and weekly summary metric is editable in this template.", categoryLabel: "Ops", }, }; @@ -505,7 +528,9 @@ const selectedEventDescription = computed(() => { }); const placeholderList = computed(() => { - const combined = [...placeholders.value, ...fallbackPlaceholders]; + const combined = placeholders.value.length + ? placeholders.value + : fallbackPlaceholders; return Array.from( new Set( combined From 2cff4bf169160369cd0a45010ac85778b44b778c Mon Sep 17 00:00:00 2001 From: Jlypx Date: Sun, 19 Jul 2026 11:39:20 +0800 Subject: [PATCH 04/59] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20golangci-lint?= =?UTF-8?q?=20=E6=A3=80=E6=9F=A5=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/notification_email_service_test.go | 2 +- .../internal/service/ops_scheduled_report_service.go | 11 +++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/backend/internal/service/notification_email_service_test.go b/backend/internal/service/notification_email_service_test.go index 975436670f..772eb45e9e 100644 --- a/backend/internal/service/notification_email_service_test.go +++ b/backend/internal/service/notification_email_service_test.go @@ -665,7 +665,7 @@ func (s *notificationEmailTestSMTPServer) handleConn(conn net.Conn) { if strings.TrimRight(dataLine, "\r\n") == "." { break } - message.WriteString(dataLine) + _, _ = message.WriteString(dataLine) } s.messageMu.Lock() s.messageBodies = append(s.messageBodies, message.String()) diff --git a/backend/internal/service/ops_scheduled_report_service.go b/backend/internal/service/ops_scheduled_report_service.go index 2ddb20c1db..eba02df40d 100644 --- a/backend/internal/service/ops_scheduled_report_service.go +++ b/backend/internal/service/ops_scheduled_report_service.go @@ -537,13 +537,13 @@ func formatOpsReportInteger(value int64) string { var builder strings.Builder builder.Grow(len(raw) + (len(raw)-start-1)/3) - builder.WriteString(raw[:start]) + _, _ = builder.WriteString(raw[:start]) digitLen := len(raw) - start for offset := 0; offset < digitLen; offset++ { if offset > 0 && (digitLen-offset)%3 == 0 { - builder.WriteByte(',') + _ = builder.WriteByte(',') } - builder.WriteByte(raw[start+offset]) + _ = builder.WriteByte(raw[start+offset]) } return builder.String() } @@ -625,11 +625,6 @@ func (s *OpsScheduledReportService) generateReportContent(ctx context.Context, r } } -func (s *OpsScheduledReportService) generateReportHTML(ctx context.Context, report *opsScheduledReport, now time.Time) (string, error) { - content, err := s.generateReportContent(ctx, report, now) - return content.html, err -} - func buildOpsSummaryEmailHTML(title string, start, end time.Time, overview *OpsDashboardOverview) string { if overview == nil { return fmt.Sprintf("

%s

No data.

", htmlEscape(title)) From 1433fbc44d378cef3d4757d8f9b40137de44ecee Mon Sep 17 00:00:00 2001 From: superman2003 <2112076433zcr@gmail.com> Date: Sun, 19 Jul 2026 12:17:23 +0800 Subject: [PATCH 05/59] fix(grok): estimate messages count tokens locally --- .../handler/openai_gateway_count_tokens.go | 42 +++++++++++++++++ backend/internal/server/routes/gateway.go | 34 ++++++-------- .../internal/server/routes/gateway_test.go | 28 +++++++++--- .../service/openai_gateway_count_tokens.go | 33 ++++++++++++++ .../openai_gateway_count_tokens_test.go | 45 +++++++++++++++++++ 5 files changed, 155 insertions(+), 27 deletions(-) diff --git a/backend/internal/handler/openai_gateway_count_tokens.go b/backend/internal/handler/openai_gateway_count_tokens.go index a9c2710b9d..bfbd2d72a5 100644 --- a/backend/internal/handler/openai_gateway_count_tokens.go +++ b/backend/internal/handler/openai_gateway_count_tokens.go @@ -13,6 +13,48 @@ import ( "go.uber.org/zap" ) +// GrokCountTokens handles Anthropic-compatible count_tokens requests locally. +// The route middleware already authenticates the API key and resolves the +// group; this handler intentionally does not select an account or check billing. +func (h *OpenAIGatewayHandler) GrokCountTokens(c *gin.Context) { + body, err := readLenientJSONRequestBodyWithPrealloc(c.Request, h.cfg) + if err != nil { + if maxErr, ok := extractMaxBytesError(err); ok { + h.anthropicErrorResponse(c, http.StatusRequestEntityTooLarge, "invalid_request_error", buildBodyTooLargeMessage(maxErr.Limit)) + return + } + h.anthropicErrorResponse(c, http.StatusBadRequest, "invalid_request_error", "Failed to read request body") + return + } + if len(body) == 0 { + h.anthropicErrorResponse(c, http.StatusBadRequest, "invalid_request_error", "Request body is empty") + return + } + + bodyRef := service.NewRequestBodyRef(body) + parsedReq, err := service.ParseGatewayRequest(bodyRef, domain.PlatformAnthropic) + if err != nil { + logRequestBodyParseFailure(requestLogger(c, "handler.openai_gateway.grok_count_tokens"), body, err) + h.anthropicErrorResponse(c, http.StatusBadRequest, "invalid_request_error", "Failed to parse request body") + return + } + if parsedReq.Model == "" { + h.anthropicErrorResponse(c, http.StatusBadRequest, "invalid_request_error", "model is required") + return + } + + estimated, err := service.EstimateGrokCountTokens(parsedReq.Body.Bytes()) + if err != nil { + requestLogger(c, "handler.openai_gateway.grok_count_tokens").Warn("grok_count_tokens.local_estimate_failed", zap.Error(err)) + h.anthropicErrorResponse(c, http.StatusBadRequest, "invalid_request_error", "Failed to parse request body") + return + } + + setOpsRequestContext(c, parsedReq.Model, false) + setOpsEndpointContext(c, "", int16(service.RequestTypeFromLegacy(false, false))) + c.JSON(http.StatusOK, gin.H{"input_tokens": estimated}) +} + // CountTokens handles Anthropic-compatible POST /v1/messages/count_tokens for OpenAI groups. // It validates billing and routes to an OpenAI token-count bridge without taking concurrency slots // or recording usage. diff --git a/backend/internal/server/routes/gateway.go b/backend/internal/server/routes/gateway.go index 07b8f1ebd8..8e14f1d5aa 100644 --- a/backend/internal/server/routes/gateway.go +++ b/backend/internal/server/routes/gateway.go @@ -43,6 +43,16 @@ func RegisterGatewayRoutes( isOpenAIGatewayPlatform := func(c *gin.Context) bool { return getGroupPlatform(c) == service.PlatformOpenAI } + countTokensHandler := func(c *gin.Context) { + switch getGroupPlatform(c) { + case service.PlatformOpenAI: + h.OpenAIGateway.CountTokens(c) + case service.PlatformGrok: + h.OpenAIGateway.GrokCountTokens(c) + default: + h.Gateway.CountTokens(c) + } + } modelsHandler := func(c *gin.Context) { if isOpenAIGatewayPlatform(c) && c.Query("client_version") != "" { h.OpenAIGateway.CodexModels(c) @@ -139,26 +149,9 @@ func RegisterGatewayRoutes( } h.Gateway.Messages(c) }) - // /v1/messages/count_tokens: OpenAI uses Anthropic-compat bridge; other - // OpenAI-compatible platforms keep the prior unsupported response. - gateway.POST("/messages/count_tokens", func(c *gin.Context) { - if isOpenAIGatewayPlatform(c) { - h.OpenAIGateway.CountTokens(c) - return - } - if isOpenAIResponsesCompatibleGatewayPlatform(c) { - service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalFeatureGate) - c.JSON(http.StatusNotFound, gin.H{ - "type": "error", - "error": gin.H{ - "type": "not_found_error", - "message": "Token counting is not supported for this platform", - }, - }) - return - } - h.Gateway.CountTokens(c) - }) + // /v1/messages/count_tokens: OpenAI bridges upstream, Grok estimates + // locally, and Anthropic-compatible platforms retain their existing path. + gateway.POST("/messages/count_tokens", countTokensHandler) // Codex CLI / Codex app refresh their model picker from the provider's // /models endpoint with a client_version query and expect the ChatGPT // Codex manifest format; other clients keep the OpenAI-style list. @@ -256,6 +249,7 @@ func RegisterGatewayRoutes( h.OpenAIGateway.ResponsesWebSocket(c) }) r.GET("/models", bodyLimit, clientRequestID, opsErrorLogger, endpointNorm, gin.HandlerFunc(apiKeyAuth), requireGroupAnthropic, modelsHandler) + r.POST("/messages/count_tokens", bodyLimit, clientRequestID, opsErrorLogger, endpointNorm, gin.HandlerFunc(apiKeyAuth), requireGroupAnthropic, countTokensHandler) codexDirect := r.Group("/backend-api/codex") codexDirect.Use(bodyLimit, clientRequestID, opsErrorLogger, endpointNorm, gin.HandlerFunc(apiKeyAuth), requireGroupAnthropic) { diff --git a/backend/internal/server/routes/gateway_test.go b/backend/internal/server/routes/gateway_test.go index 80e319db76..43a43ff787 100644 --- a/backend/internal/server/routes/gateway_test.go +++ b/backend/internal/server/routes/gateway_test.go @@ -1,6 +1,7 @@ package routes import ( + "encoding/json" "net/http" "net/http/httptest" "strings" @@ -15,6 +16,10 @@ import ( ) func newGatewayRoutesTestRouter(platform ...string) *gin.Engine { + return newGatewayRoutesTestRouterWithConfig(&config.Config{}, platform...) +} + +func newGatewayRoutesTestRouterWithConfig(cfg *config.Config, platform ...string) *gin.Engine { gin.SetMode(gin.TestMode) router := gin.New() @@ -42,7 +47,7 @@ func newGatewayRoutesTestRouter(platform ...string) *gin.Engine { nil, nil, nil, - &config.Config{}, + cfg, ) return router @@ -224,13 +229,22 @@ func TestGatewayRoutesGrokAllowsCLICompatibilityEntrypoints(t *testing.T) { require.NotContains(t, w.Body.String(), "not supported for Grok groups") } - req := httptest.NewRequest(http.MethodPost, "/v1/messages/count_tokens", strings.NewReader(`{"model":"grok","messages":[{"role":"user","content":"hi"}]}`)) - req.Header.Set("Content-Type", "application/json") - w := httptest.NewRecorder() + countTokensRouter := newGatewayRoutesTestRouterWithConfig(&config.Config{ + Gateway: config.GatewayConfig{MaxBodySize: 1024 * 1024}, + }, service.PlatformGrok) + for _, path := range []string{"/v1/messages/count_tokens", "/messages/count_tokens"} { + req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{"model":"grok","messages":[{"role":"user","content":"hi"}]}`)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() - router.ServeHTTP(w, req) - require.Equal(t, http.StatusNotFound, w.Code) - require.Contains(t, w.Body.String(), "Token counting is not supported for this platform") + countTokensRouter.ServeHTTP(w, req) + require.Equal(t, http.StatusOK, w.Code, "path=%s", path) + var response struct { + InputTokens int `json:"input_tokens"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response), "path=%s", path) + require.Positive(t, response.InputTokens, "path=%s", path) + } for _, path := range []string{ "/v1/responses", diff --git a/backend/internal/service/openai_gateway_count_tokens.go b/backend/internal/service/openai_gateway_count_tokens.go index 46b5a3843a..eccd1c4146 100644 --- a/backend/internal/service/openai_gateway_count_tokens.go +++ b/backend/internal/service/openai_gateway_count_tokens.go @@ -39,6 +39,39 @@ type openAIInputTokensCountPrepared struct { UpstreamModel string } +// EstimateGrokCountTokens estimates an Anthropic-compatible count_tokens request +// locally. Grok does not expose a compatible token-counting endpoint, so this +// path deliberately avoids account selection, credentials, and upstream calls. +func EstimateGrokCountTokens(body []byte) (int, error) { + var anthropicReq apicompat.AnthropicRequest + if err := json.Unmarshal(body, &anthropicReq); err != nil { + return 0, fmt.Errorf("parse anthropic count_tokens request: %w", err) + } + if strings.TrimSpace(anthropicReq.Model) == "" { + return 0, fmt.Errorf("parse anthropic count_tokens request: model is required") + } + + responsesReq, err := apicompat.AnthropicToResponses(&anthropicReq) + if err != nil { + return 0, fmt.Errorf("convert anthropic request to responses: %w", err) + } + + estimated, err := estimateOpenAIInputTokens(openAIInputTokensCountRequest{ + Model: anthropicReq.Model, + Instructions: responsesReq.Instructions, + Input: responsesReq.Input, + Tools: responsesReq.Tools, + ToolChoice: responsesReq.ToolChoice, + }) + if err != nil { + return 0, fmt.Errorf("estimate grok input tokens: %w", err) + } + if estimated < openAIInputTokensFallbackMinimum { + estimated = openAIInputTokensFallbackMinimum + } + return estimated, nil +} + // ForwardCountTokensAsAnthropic bridges Anthropic /v1/messages/count_tokens to // OpenAI POST /v1/responses/input_tokens and returns Anthropic-compatible output. func (s *OpenAIGatewayService) ForwardCountTokensAsAnthropic( diff --git a/backend/internal/service/openai_gateway_count_tokens_test.go b/backend/internal/service/openai_gateway_count_tokens_test.go index 3ab0178a0d..9719c9b3f3 100644 --- a/backend/internal/service/openai_gateway_count_tokens_test.go +++ b/backend/internal/service/openai_gateway_count_tokens_test.go @@ -236,6 +236,51 @@ func TestEstimateOpenAIInputTokens_RequestSamples(t *testing.T) { } } +func TestEstimateGrokCountTokens_AnthropicRequests(t *testing.T) { + cases := []struct { + name string + body string + }{ + { + name: "simple message", + body: `{"model":"grok-4","messages":[{"role":"user","content":"hello world"}]}`, + }, + { + name: "system blocks and tools", + body: `{ + "model":"grok-4", + "system":[{"type":"text","text":"You are helpful."}], + "messages":[{"role":"user","content":[{"type":"text","text":"look up the weather"}]}], + "tools":[{"name":"lookup_weather","description":"Look up weather","input_schema":{"type":"object","properties":{"city":{"type":"string"}}}}], + "tool_choice":{"type":"auto"} + }`, + }, + { + name: "empty conversation uses positive minimum", + body: `{"model":"grok-4","messages":[]}`, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + got, err := EstimateGrokCountTokens([]byte(tt.body)) + require.NoError(t, err) + require.Positive(t, got) + }) + } +} + +func TestEstimateGrokCountTokens_RejectsInvalidRequests(t *testing.T) { + for _, body := range []string{ + `{`, + `{"messages":[{"role":"user","content":"hello"}]}`, + `{"model":"grok-4","messages":[{"role":"user","content":{"unexpected":true}}]}`, + } { + _, err := EstimateGrokCountTokens([]byte(body)) + require.Error(t, err, "body=%s", body) + } +} + func TestOpenAIInputTokensEncodingForModel(t *testing.T) { cases := []struct { model string From b8c640d218471321d1bbaf1f1ba1bb5a5b50154a Mon Sep 17 00:00:00 2001 From: superman2003 <2112076433zcr@gmail.com> Date: Sun, 19 Jul 2026 12:17:31 +0800 Subject: [PATCH 06/59] fix(proxy): include Grok in quality checks --- backend/internal/service/admin_service.go | 8 ++++ .../admin_service_proxy_quality_test.go | 41 +++++++++++++++++++ frontend/src/views/admin/ProxiesView.vue | 20 +++++---- 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/backend/internal/service/admin_service.go b/backend/internal/service/admin_service.go index 013cb04049..023c6acc1f 100644 --- a/backend/internal/service/admin_service.go +++ b/backend/internal/service/admin_service.go @@ -576,6 +576,14 @@ var proxyQualityTargets = []proxyQualityTarget{ http.StatusOK: {}, }, }, + { + Target: "grok", + URL: "https://api.x.ai/v1/models", + Method: http.MethodGet, + AllowedStatuses: map[int]struct{}{ + http.StatusUnauthorized: {}, + }, + }, } const ( diff --git a/backend/internal/service/admin_service_proxy_quality_test.go b/backend/internal/service/admin_service_proxy_quality_test.go index 023b448fb5..2347c2deba 100644 --- a/backend/internal/service/admin_service_proxy_quality_test.go +++ b/backend/internal/service/admin_service_proxy_quality_test.go @@ -93,3 +93,44 @@ func TestRunProxyQualityTarget_AllowedStatusPassForUnauthorized(t *testing.T) { require.Equal(t, http.StatusUnauthorized, item.HTTPStatus) require.Contains(t, item.Message, "目标可达") } + +func TestProxyQualityTargets_IncludesGrok(t *testing.T) { + var grokTarget *proxyQualityTarget + for i := range proxyQualityTargets { + if proxyQualityTargets[i].Target == "grok" { + grokTarget = &proxyQualityTargets[i] + break + } + } + + require.NotNil(t, grokTarget) + require.Equal(t, "https://api.x.ai/v1/models", grokTarget.URL) + require.Equal(t, http.MethodGet, grokTarget.Method) + require.Contains(t, grokTarget.AllowedStatuses, http.StatusUnauthorized) +} + +func TestRunProxyQualityTarget_GrokUnauthorizedPasses(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + t.Errorf("unexpected method: %s", r.Method) + } + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte(`{"error":"unauthorized"}`)) + })) + defer server.Close() + + target := proxyQualityTarget{ + Target: "grok", + URL: server.URL, + Method: http.MethodGet, + AllowedStatuses: map[int]struct{}{ + http.StatusUnauthorized: {}, + }, + } + + item := runProxyQualityTarget(context.Background(), server.Client(), target) + require.Equal(t, "grok", item.Target) + require.Equal(t, "pass", item.Status) + require.Equal(t, http.StatusUnauthorized, item.HTTPStatus) + require.Contains(t, item.Message, "目标可达") +} diff --git a/frontend/src/views/admin/ProxiesView.vue b/frontend/src/views/admin/ProxiesView.vue index 3cf86b52fa..0572cc254b 100644 --- a/frontend/src/views/admin/ProxiesView.vue +++ b/frontend/src/views/admin/ProxiesView.vue @@ -881,21 +881,21 @@ - - - - + + + + - - + - - + - - + diff --git a/README_CN.md b/README_CN.md index bc33ad82a0..33e35e79d5 100644 --- a/README_CN.md +++ b/README_CN.md @@ -80,8 +80,8 @@ - - + diff --git a/README_JA.md b/README_JA.md index 90541cadf4..0506dfb594 100644 --- a/README_JA.md +++ b/README_JA.md @@ -79,8 +79,8 @@ - - + From 2518691dbcc156d928f1c0dab4ac339cb6893fc8 Mon Sep 17 00:00:00 2001 From: shaw Date: Mon, 20 Jul 2026 09:41:02 +0800 Subject: [PATCH 58/59] =?UTF-8?q?fix(config):=20=E4=B8=BA=20trusted=5Fprox?= =?UTF-8?q?ies=20=E4=B8=8E=20forwarded=5Fclient=5Fip=5Fheaders=20=E6=B3=A8?= =?UTF-8?q?=E5=86=8C=20BindEnv=EF=BC=8C=E4=BF=AE=E5=A4=8D=20env=20?= =?UTF-8?q?=E5=8F=AF=E8=BE=BE=E6=80=A7=E5=AE=88=E5=8D=AB=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #4593 引入的 TestConfigKeysAreEnvReachable 与 #4604 新增的两个配置键语义相撞： 两键在 setDefaults 中无注册，守卫测试判定其环境变量不可达。 不能按守卫注释直接 SetDefault：viper 的 IsSet 会连同 defaults 一起上报， 任何已注册默认值都会让 load() 中 trustedProxiesConfigured 的显式配置探测 永远为真，摧毁 #4600 的 absent/empty 区分。改用 BindEnv 注册——绑定键同样 进入 AllKeys()（守卫测试与 Unmarshal 均可见），而变量缺席时不影响 IsSet。 两个环境变量本就由 load() 中 os.LookupEnv 手工解析，行为不变。 --- backend/internal/config/config.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index d489136ed6..c9f2c02c4a 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -2374,6 +2374,18 @@ func setEnvReachableDefaults() { viper.SetDefault("gateway.openai_scheduler.sticky_escape_error_rate", 0.0) viper.SetDefault("gateway.openai_scheduler.sticky_escape_ttft_ms", 0) + // server.trusted_proxies and security.forwarded_client_ip_headers are the + // other exception: load() distinguishes explicit configuration from absence + // (issue #4600), and viper.IsSet also reports registered defaults, so a + // SetDefault would make trusted proxies look permanently configured. Both + // environment variables are parsed by hand in load() via os.LookupEnv and + // were never silently dropped; binding them here records that reachability + // where AllKeys() — and the env-reachability guard — can see it, without + // affecting IsSet while the variables are absent. BindEnv only errors when + // called without arguments. + _ = viper.BindEnv("server.trusted_proxies", "SERVER_TRUSTED_PROXIES") + _ = viper.BindEnv("security.forwarded_client_ip_headers", "SECURITY_FORWARDED_CLIENT_IP_HEADERS") + // Third-party login providers. These carry client secrets and are exactly // the settings an operator expects to inject via the environment, but every // key here was previously unreachable that way. From 97afb4089c67f5ccbfd594c481c1ef364b0b4f9a Mon Sep 17 00:00:00 2001 From: shaw Date: Mon, 20 Jul 2026 10:46:37 +0800 Subject: [PATCH 59/59] fix(lint): check strings.Builder WriteString return value (errcheck) --- backend/internal/service/openai_codex_transform.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/internal/service/openai_codex_transform.go b/backend/internal/service/openai_codex_transform.go index 9d47c1c56e..c2169ab656 100644 --- a/backend/internal/service/openai_codex_transform.go +++ b/backend/internal/service/openai_codex_transform.go @@ -1176,7 +1176,7 @@ func extractLosslessTextFromContent(content any) (string, bool) { if !ok { return "", false } - b.WriteString(text) + _, _ = b.WriteString(text) } return b.String(), true default:
{{ t('admin.proxies.qualityTableTarget') }}{{ t('admin.proxies.qualityTableStatus') }}HTTP{{ t('admin.proxies.qualityTableLatency') }}{{ t('admin.proxies.qualityTableTarget') }}{{ t('admin.proxies.qualityTableStatus') }}HTTP{{ t('admin.proxies.qualityTableLatency') }} {{ t('admin.proxies.qualityTableMessage') }}
{{ qualityTargetLabel(item.target) }} - {{ qualityStatusLabel(item.status) }} + {{ qualityTargetLabel(item.target) }} + {{ qualityStatusLabel(item.status) }} {{ item.http_status ?? '-' }} + {{ item.http_status ?? '-' }} {{ typeof item.latency_ms === 'number' ? `${item.latency_ms}ms` : '-' }} @@ -1777,6 +1777,8 @@ const qualityTargetLabel = (target: string) => { return 'Anthropic' case 'gemini': return 'Gemini' + case 'grok': + return 'Grok' default: return target } From c99e479bcc14de1e154c901b1ad75d71c439a1f2 Mon Sep 17 00:00:00 2001 From: superman2003 <2112076433zcr@gmail.com> Date: Sun, 19 Jul 2026 12:17:42 +0800 Subject: [PATCH 07/59] fix(grok): retry transient billing probes --- .../internal/service/grok_quota_service.go | 80 +++++--- .../service/grok_quota_service_test.go | 172 ++++++++++++++++++ 2 files changed, 228 insertions(+), 24 deletions(-) diff --git a/backend/internal/service/grok_quota_service.go b/backend/internal/service/grok_quota_service.go index cf28f6c6ec..2c703f9f10 100644 --- a/backend/internal/service/grok_quota_service.go +++ b/backend/internal/service/grok_quota_service.go @@ -23,6 +23,8 @@ const ( grokQuotaProbeInput = "hi" grokQuotaDefaultModel = grokDefaultResponsesModel grokBillingExtraKey = "grok_billing_snapshot" + grokBillingMaxAttempts = 2 + grokBillingRetryDelay = 100 * time.Millisecond ) type GrokQuotaProbeResult struct { @@ -356,33 +358,63 @@ func (s *GrokQuotaService) fetchBilling( if err != nil { return nil, 0, infraerrors.Newf(http.StatusBadRequest, "GROK_QUOTA_BASE_URL_INVALID", "invalid Grok base_url: %v", err) } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, billingURL, nil) - if err != nil { - return nil, 0, infraerrors.Newf(http.StatusInternalServerError, "GROK_QUOTA_PROBE_REQUEST_BUILD_FAILED", "failed to build billing request: %v", err) - } - xai.ApplyCLIBillingHeaders(req, token) - // billing 探测与真实转发保持同一套账号级请求头覆写。 - account.ApplyHeaderOverrides(req.Header) - resp, err := s.httpUpstream.Do(req, proxyURL, account.ID, maxInt(account.Concurrency, 2)) - if err != nil { - return nil, 0, infraerrors.Newf(http.StatusBadGateway, "GROK_QUOTA_PROBE_REQUEST_FAILED", "billing request failed: %v", err) - } - defer func() { _ = resp.Body.Close() }() + for attempt := 0; attempt < grokBillingMaxAttempts; attempt++ { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, billingURL, nil) + if err != nil { + return nil, 0, infraerrors.Newf(http.StatusInternalServerError, "GROK_QUOTA_PROBE_REQUEST_BUILD_FAILED", "failed to build billing request: %v", err) + } + xai.ApplyCLIBillingHeaders(req, token) + // billing 探测与真实转发保持同一套账号级请求头覆写。 + account.ApplyHeaderOverrides(req.Header) + resp, requestErr := s.httpUpstream.Do(req, proxyURL, account.ID, maxInt(account.Concurrency, 2)) - bodyBytes, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) - if resp.StatusCode == http.StatusTooManyRequests { - return nil, resp.StatusCode, nil + statusCode := 0 + var bodyBytes []byte + if requestErr == nil { + statusCode = resp.StatusCode + bodyBytes, _ = io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + _ = resp.Body.Close() + } + + shouldRetry := requestErr != nil || isRetryableGrokBillingStatus(statusCode) + if shouldRetry && attempt+1 < grokBillingMaxAttempts { + timer := time.NewTimer(grokBillingRetryDelay) + select { + case <-timer.C: + case <-ctx.Done(): + timer.Stop() + return nil, statusCode, infraerrors.Newf(http.StatusBadGateway, "GROK_QUOTA_PROBE_REQUEST_FAILED", "billing request failed: %v", ctx.Err()) + } + continue + } + + if requestErr != nil { + return nil, 0, infraerrors.Newf(http.StatusBadGateway, "GROK_QUOTA_PROBE_REQUEST_FAILED", "billing request failed: %v", requestErr) + } + if statusCode == http.StatusTooManyRequests { + return nil, statusCode, nil + } + if statusCode >= 400 { + bodyText := truncate(strings.TrimSpace(string(bodyBytes)), 240) + slog.Warn("grok_quota_billing_failed", "account_id", account.ID, "weekly", weekly, "status", statusCode, "body", bodyText) + return nil, statusCode, infraerrors.Newf(mapUpstreamStatus(statusCode), "GROK_QUOTA_PROBE_UPSTREAM_ERROR", "billing returned %d: %s", statusCode, bodyText) + } + payload, err := xai.ParseBillingPayload(bodyBytes) + if err != nil { + return nil, statusCode, infraerrors.Newf(http.StatusBadGateway, "GROK_QUOTA_BILLING_PARSE_ERROR", "failed to parse billing body: %v", err) + } + return xai.BuildBillingSummary(payload.Config), statusCode, nil } - if resp.StatusCode >= 400 { - bodyText := truncate(strings.TrimSpace(string(bodyBytes)), 240) - slog.Warn("grok_quota_billing_failed", "account_id", account.ID, "weekly", weekly, "status", resp.StatusCode, "body", bodyText) - return nil, resp.StatusCode, infraerrors.Newf(mapUpstreamStatus(resp.StatusCode), "GROK_QUOTA_PROBE_UPSTREAM_ERROR", "billing returned %d: %s", resp.StatusCode, bodyText) + return nil, 0, infraerrors.New(http.StatusBadGateway, "GROK_QUOTA_PROBE_REQUEST_FAILED", "billing request failed") +} + +func isRetryableGrokBillingStatus(statusCode int) bool { + switch statusCode { + case http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout: + return true + default: + return false } - payload, err := xai.ParseBillingPayload(bodyBytes) - if err != nil { - return nil, resp.StatusCode, infraerrors.Newf(http.StatusBadGateway, "GROK_QUOTA_BILLING_PARSE_ERROR", "failed to parse billing body: %v", err) - } - return xai.BuildBillingSummary(payload.Config), resp.StatusCode, nil } func mergeGrokBillingProbeErrors(weeklyStatus, monthlyStatus int, weeklyErr, monthlyErr error) error { diff --git a/backend/internal/service/grok_quota_service_test.go b/backend/internal/service/grok_quota_service_test.go index ae3dacc4a7..9ffb70f000 100644 --- a/backend/internal/service/grok_quota_service_test.go +++ b/backend/internal/service/grok_quota_service_test.go @@ -5,6 +5,7 @@ package service import ( "bytes" "context" + "errors" "io" "log/slog" "net/http" @@ -127,6 +128,44 @@ type grokHybridUpstream struct { billingHeaders http.Header } +type grokQuotaUpstreamStep struct { + status int + body string + err error +} + +type grokQuotaSequenceUpstream struct { + httpUpstreamRecorder + mu sync.Mutex + steps []grokQuotaUpstreamStep + requests []*http.Request +} + +func (u *grokQuotaSequenceUpstream) Do(req *http.Request, _ string, _ int64, _ int) (*http.Response, error) { + u.mu.Lock() + defer u.mu.Unlock() + u.requests = append(u.requests, req) + index := len(u.requests) - 1 + if index >= len(u.steps) { + return nil, errors.New("unexpected upstream request") + } + step := u.steps[index] + if step.err != nil { + return nil, step.err + } + return &http.Response{ + StatusCode: step.status, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(step.body)), + }, nil +} + +func (u *grokQuotaSequenceUpstream) snapshotRequests() []*http.Request { + u.mu.Lock() + defer u.mu.Unlock() + return append([]*http.Request(nil), u.requests...) +} + func (u *grokHybridUpstream) Do(req *http.Request, _ string, _ int64, _ int) (*http.Response, error) { var body []byte if req != nil && req.Body != nil { @@ -228,6 +267,139 @@ func healthyGrokQuotaOAuthAccount(id int64) *Account { } } +func TestGrokQuotaServiceFetchBillingRetries502ThenSucceeds(t *testing.T) { + account := healthyGrokQuotaOAuthAccount(401) + upstream := &grokQuotaSequenceUpstream{steps: []grokQuotaUpstreamStep{ + {status: http.StatusBadGateway, body: `The origin web server returned an invalid or incomplete response to Cloudflare.`}, + {status: http.StatusOK, body: `{"config":{"currentPeriod":{"type":"WEEKLY","start":"2026-07-09T03:25:00Z","end":"2026-07-16T03:25:00Z"},"creditUsagePercent":12}}`}, + }} + svc := &GrokQuotaService{httpUpstream: upstream} + + summary, status, err := svc.fetchBilling(context.Background(), account, "access-token", "", true) + + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + require.NotNil(t, summary) + require.NotNil(t, summary.UsagePercent) + require.Equal(t, 12.0, *summary.UsagePercent) + requests := upstream.snapshotRequests() + require.Len(t, requests, 2) + require.Equal(t, http.MethodGet, requests[0].Method) + require.Equal(t, "/v1/billing", requests[0].URL.Path) + require.Equal(t, "format=credits", requests[0].URL.RawQuery) +} + +func TestGrokQuotaServiceFetchBillingRetriesTransportErrorThenSucceeds(t *testing.T) { + account := healthyGrokQuotaOAuthAccount(402) + upstream := &grokQuotaSequenceUpstream{steps: []grokQuotaUpstreamStep{ + {err: errors.New("temporary transport failure")}, + {status: http.StatusOK, body: `{"config":{"currentPeriod":{"type":"WEEKLY"},"creditUsagePercent":8}}`}, + }} + svc := &GrokQuotaService{httpUpstream: upstream} + + summary, status, err := svc.fetchBilling(context.Background(), account, "access-token", "", true) + + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + require.NotNil(t, summary) + require.Len(t, upstream.snapshotRequests(), 2) +} + +func TestGrokQuotaServiceFetchBillingStopsAfterSingleTransientRetry(t *testing.T) { + account := healthyGrokQuotaOAuthAccount(403) + upstream := &grokQuotaSequenceUpstream{steps: []grokQuotaUpstreamStep{ + {status: http.StatusBadGateway, body: `cloudflare failure`}, + {status: http.StatusBadGateway, body: `cloudflare failure`}, + {status: http.StatusOK, body: `{"config":{"currentPeriod":{"type":"WEEKLY"}}}`}, + }} + svc := &GrokQuotaService{httpUpstream: upstream} + + summary, status, err := svc.fetchBilling(context.Background(), account, "access-token", "", true) + + require.Error(t, err) + require.Nil(t, summary) + require.Equal(t, http.StatusBadGateway, status) + require.Equal(t, "GROK_QUOTA_PROBE_UPSTREAM_ERROR", infraerrors.Reason(err)) + require.Contains(t, infraerrors.Message(err), "billing returned 502: cloudflare failure") + require.Len(t, upstream.snapshotRequests(), 2) +} + +func TestGrokQuotaServiceFetchBillingDoesNotRetryNonTransientStatuses(t *testing.T) { + tests := []struct { + name string + status int + wantErr bool + }{ + {name: "unauthorized", status: http.StatusUnauthorized, wantErr: true}, + {name: "forbidden", status: http.StatusForbidden, wantErr: true}, + {name: "rate limited", status: http.StatusTooManyRequests, wantErr: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + account := healthyGrokQuotaOAuthAccount(404) + upstream := &grokQuotaSequenceUpstream{steps: []grokQuotaUpstreamStep{ + {status: tt.status, body: `{"error":{"message":"rejected"}}`}, + {status: http.StatusOK, body: `{"config":{"currentPeriod":{"type":"WEEKLY"}}}`}, + }} + svc := &GrokQuotaService{httpUpstream: upstream} + + summary, status, err := svc.fetchBilling(context.Background(), account, "access-token", "", true) + + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.Nil(t, summary) + require.Equal(t, tt.status, status) + require.Len(t, upstream.snapshotRequests(), 1) + }) + } +} + +func TestIsRetryableGrokBillingStatus(t *testing.T) { + tests := []struct { + status int + want bool + }{ + {status: http.StatusBadGateway, want: true}, + {status: http.StatusServiceUnavailable, want: true}, + {status: http.StatusGatewayTimeout, want: true}, + {status: http.StatusUnauthorized, want: false}, + {status: http.StatusForbidden, want: false}, + {status: http.StatusTooManyRequests, want: false}, + } + + for _, tt := range tests { + t.Run(strconv.Itoa(tt.status), func(t *testing.T) { + require.Equal(t, tt.want, isRetryableGrokBillingStatus(tt.status)) + }) + } +} + +func TestGrokQuotaServiceProbeUsageDoesNotRetryResponsesPost(t *testing.T) { + account := healthyGrokQuotaOAuthAccount(405) + repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{ + accountsByID: map[int64]*Account{account.ID: account}, + }} + upstream := &grokQuotaSequenceUpstream{steps: []grokQuotaUpstreamStep{ + {status: http.StatusBadGateway, body: `cloudflare failure`}, + {status: http.StatusOK, body: `{"id":"unexpected_retry"}`}, + }} + svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), upstream, nil) + + result, err := svc.ProbeUsage(context.Background(), account.ID) + + require.Error(t, err) + require.Nil(t, result) + require.Equal(t, "GROK_QUOTA_PROBE_UPSTREAM_ERROR", infraerrors.Reason(err)) + requests := upstream.snapshotRequests() + require.Len(t, requests, 1) + require.Equal(t, http.MethodPost, requests[0].Method) + require.Equal(t, "/v1/responses", requests[0].URL.Path) +} + func TestGrokQuotaServiceProbeUsageStoresHeaders(t *testing.T) { t.Parallel() From d2753dc2e29037bf502396cf2272601c87aec209 Mon Sep 17 00:00:00 2001 From: superman2003 <2112076433zcr@gmail.com> Date: Sun, 19 Jul 2026 12:17:52 +0800 Subject: [PATCH 08/59] fix(grok): serve current free quota policy --- backend/internal/pkg/xai/quota.go | 12 +++++++++ backend/internal/pkg/xai/quota_test.go | 8 ++++++ .../internal/service/account_usage_service.go | 1 + .../internal/service/grok_quota_fetcher.go | 5 ++-- .../service/grok_quota_fetcher_test.go | 1 + .../service/openai_gateway_grok_cache.go | 4 +-- .../service/openai_gateway_grok_cache_test.go | 17 ++++++++++-- .../components/account/AccountUsageCell.vue | 8 +++--- .../__tests__/AccountUsageCell.spec.ts | 27 +++++++++++-------- .../src/i18n/locales/en/admin/accounts.ts | 2 +- .../src/i18n/locales/zh/admin/accounts.ts | 2 +- frontend/src/types/index.ts | 1 + 12 files changed, 65 insertions(+), 23 deletions(-) diff --git a/backend/internal/pkg/xai/quota.go b/backend/internal/pkg/xai/quota.go index 1387c5bdad..fe269d1d84 100644 --- a/backend/internal/pkg/xai/quota.go +++ b/backend/internal/pkg/xai/quota.go @@ -7,6 +7,18 @@ import ( "time" ) +const GrokFreeRolling24hTokenLimit int64 = 1_000_000 + +var grokFreeRolling24hTokenLimits = map[int64]struct{}{ + GrokFreeRolling24hTokenLimit: {}, + 2_000_000: {}, // Legacy Free limit observed before July 2026. +} + +func IsGrokFreeRolling24hTokenLimit(limit int64) bool { + _, ok := grokFreeRolling24hTokenLimits[limit] + return ok +} + type QuotaWindow struct { Limit *int64 `json:"limit,omitempty"` Remaining *int64 `json:"remaining,omitempty"` diff --git a/backend/internal/pkg/xai/quota_test.go b/backend/internal/pkg/xai/quota_test.go index 593edc8eb9..983a87e84f 100644 --- a/backend/internal/pkg/xai/quota_test.go +++ b/backend/internal/pkg/xai/quota_test.go @@ -61,3 +61,11 @@ func TestObserveQuotaHeadersRecordsNoHeaderProbe(t *testing.T) { require.Nil(t, snapshot.Requests) require.Nil(t, snapshot.Tokens) } + +func TestIsGrokFreeRolling24hTokenLimit(t *testing.T) { + t.Parallel() + + require.True(t, IsGrokFreeRolling24hTokenLimit(GrokFreeRolling24hTokenLimit)) + require.True(t, IsGrokFreeRolling24hTokenLimit(2_000_000), "legacy snapshots remain classifiable") + require.False(t, IsGrokFreeRolling24hTokenLimit(3_000_000)) +} diff --git a/backend/internal/service/account_usage_service.go b/backend/internal/service/account_usage_service.go index 12dc25a673..e9502ec4c6 100644 --- a/backend/internal/service/account_usage_service.go +++ b/backend/internal/service/account_usage_service.go @@ -207,6 +207,7 @@ type UsageInfo struct { GrokLastQuotaProbeAt string `json:"grok_last_quota_probe_at,omitempty"` GrokLastHeadersSeenAt string `json:"grok_last_headers_seen_at,omitempty"` GrokLastStatusCode int `json:"grok_last_status_code,omitempty"` + GrokFreeTokenLimit int64 `json:"grok_free_token_limit,omitempty"` GrokLocalUsage *WindowStats `json:"grok_local_usage,omitempty"` GrokLocalUsage24h *WindowStats `json:"grok_local_usage_24h,omitempty"` GrokLocalUsage7d *WindowStats `json:"grok_local_usage_7d,omitempty"` diff --git a/backend/internal/service/grok_quota_fetcher.go b/backend/internal/service/grok_quota_fetcher.go index 1f2d6374ad..545b1c9e97 100644 --- a/backend/internal/service/grok_quota_fetcher.go +++ b/backend/internal/service/grok_quota_fetcher.go @@ -21,8 +21,9 @@ func NewGrokQuotaFetcher() *GrokQuotaFetcher { func (f *GrokQuotaFetcher) BuildUsageInfo(account *Account) *UsageInfo { now := time.Now() usage := &UsageInfo{ - Source: "passive", - UpdatedAt: &now, + Source: "passive", + UpdatedAt: &now, + GrokFreeTokenLimit: xai.GrokFreeRolling24hTokenLimit, } if account == nil { usage.ErrorCode = "quota_unknown" diff --git a/backend/internal/service/grok_quota_fetcher_test.go b/backend/internal/service/grok_quota_fetcher_test.go index 6bea2814f2..a54ecc6c81 100644 --- a/backend/internal/service/grok_quota_fetcher_test.go +++ b/backend/internal/service/grok_quota_fetcher_test.go @@ -19,6 +19,7 @@ func TestGrokQuotaFetcherBuildUsageInfoUnknownUntilFirstSnapshot(t *testing.T) { usage := NewGrokQuotaFetcher().BuildUsageInfo(&Account{Platform: PlatformGrok, Type: AccountTypeOAuth}) require.Equal(t, "passive", usage.Source) + require.Equal(t, xai.GrokFreeRolling24hTokenLimit, usage.GrokFreeTokenLimit) require.Equal(t, "quota_unknown", usage.ErrorCode) require.Contains(t, usage.Error, "unknown until billing is probed") } diff --git a/backend/internal/service/openai_gateway_grok_cache.go b/backend/internal/service/openai_gateway_grok_cache.go index d9e34cbb58..6705764211 100644 --- a/backend/internal/service/openai_gateway_grok_cache.go +++ b/backend/internal/service/openai_gateway_grok_cache.go @@ -6,6 +6,7 @@ import ( "net/http" "strings" + "github.com/Wei-Shaw/sub2api/internal/pkg/xai" "github.com/gin-gonic/gin" "github.com/tidwall/gjson" "github.com/tidwall/sjson" @@ -15,7 +16,6 @@ const ( grokConversationIDHeader = "X-Grok-Conv-Id" grokFreeCacheNativeToolsJSON = `[{"type":"web_search"},{"type":"x_search"}]` grokFreeCacheDisabledToolChoice = "none" - grokFreeRolling24hTokenLimit = int64(2_000_000) ) // resolveGrokCacheIdentity derives one stable, tenant-isolated routing identity @@ -187,7 +187,7 @@ func isKnownGrokFreeAccount(account *Account) bool { } } if snapshot.Tokens != nil && snapshot.Tokens.Limit != nil && - *snapshot.Tokens.Limit == grokFreeRolling24hTokenLimit { + xai.IsGrokFreeRolling24hTokenLimit(*snapshot.Tokens.Limit) { inferredFreeSignal = true } } diff --git a/backend/internal/service/openai_gateway_grok_cache_test.go b/backend/internal/service/openai_gateway_grok_cache_test.go index 73b1ea5ec1..a90ffcdbed 100644 --- a/backend/internal/service/openai_gateway_grok_cache_test.go +++ b/backend/internal/service/openai_gateway_grok_cache_test.go @@ -8,6 +8,7 @@ import ( "sync" "testing" + "github.com/Wei-Shaw/sub2api/internal/pkg/xai" "github.com/gin-gonic/gin" "github.com/stretchr/testify/require" "github.com/tidwall/gjson" @@ -325,7 +326,19 @@ func TestGrokFreeMessagesFunctionToolCacheRouteRequiresKnownFreeTier(t *testing. a := healthyGrokOAuthGatewayTestAccount(9112, "access-token") a.Extra = map[string]any{grokQuotaSnapshotExtraKey: map[string]any{ "headers_observed": true, - "tokens": map[string]any{"limit": grokFreeRolling24hTokenLimit}, + "tokens": map[string]any{"limit": xai.GrokFreeRolling24hTokenLimit}, + }} + return a + }(), + wantMix: true, + }, + { + name: "legacy free rolling token quota", + account: func() *Account { + a := healthyGrokOAuthGatewayTestAccount(9113, "access-token") + a.Extra = map[string]any{grokQuotaSnapshotExtraKey: map[string]any{ + "headers_observed": true, + "tokens": map[string]any{"limit": int64(2_000_000)}, }} return a }(), @@ -347,7 +360,7 @@ func TestGrokFreeMessagesFunctionToolCacheRouteRequiresKnownFreeTier(t *testing. grokBillingExtraKey: map[string]any{"plan": "SuperGrok", "status_code": http.StatusOK}, grokQuotaSnapshotExtraKey: map[string]any{ "headers_observed": true, - "tokens": map[string]any{"limit": grokFreeRolling24hTokenLimit}, + "tokens": map[string]any{"limit": int64(2_000_000)}, }, } return a diff --git a/frontend/src/components/account/AccountUsageCell.vue b/frontend/src/components/account/AccountUsageCell.vue index 9a3d6a86e9..9a3ac13f02 100644 --- a/frontend/src/components/account/AccountUsageCell.vue +++ b/frontend/src/components/account/AccountUsageCell.vue @@ -408,7 +408,7 @@ () const USAGE_CACHE_TTL = 5 * 60 * 1000 // 5 minutes -// xAI Free billing exposes a window without usage_percent, so estimate it from local tokens. -const GROK_FREE_TOKEN_LIMIT = 2_000_000 const props = withDefaults( defineProps<{ @@ -1112,8 +1110,10 @@ const grokLocalUsage = computed(() => { }) const grokFreeTokenBar = computed(() => { if (!grokIsFree.value || !grokFreeQuotaUsage.value) return null + const limit = usageInfo.value?.grok_free_token_limit + if (typeof limit !== 'number' || limit <= 0) return null const used = Math.max(0, grokFreeQuotaUsage.value.tokens || 0) - return { utilization: Math.min(100, (used / GROK_FREE_TOKEN_LIMIT) * 100) } + return { utilization: Math.min(100, (used / limit) * 100), limit } }) const grokQuotaUnknown = computed(() => { if (props.account.platform !== 'grok') return false diff --git a/frontend/src/components/account/__tests__/AccountUsageCell.spec.ts b/frontend/src/components/account/__tests__/AccountUsageCell.spec.ts index 5f61494c5f..203b150183 100644 --- a/frontend/src/components/account/__tests__/AccountUsageCell.spec.ts +++ b/frontend/src/components/account/__tests__/AccountUsageCell.spec.ts @@ -705,11 +705,12 @@ describe('AccountUsageCell', () => { it.each([ { tokens: 0, expected: 0, compact: '0' }, - { tokens: 1_000_000, expected: 50, compact: '1.0M' }, - { tokens: 2_000_000, expected: 100, compact: '2.0M' }, - { tokens: 2_200_000, expected: 100, compact: '2.2M' } - ])('Grok Free derives its 2M quota from local tokens: $tokens -> $expected%', async ({ tokens, expected, compact }) => { + { tokens: 500_000, expected: 50, compact: '500.0K' }, + { tokens: 1_000_000, expected: 100, compact: '1.0M' }, + { tokens: 1_100_000, expected: 100, compact: '1.1M' } + ])('Grok Free derives its 1M quota from local tokens: $tokens -> $expected%', async ({ tokens, expected, compact }) => { getUsage.mockResolvedValue({ + grok_free_token_limit: 1_000_000, grok_billing: { period_type: 'weekly', usage_percent: null, @@ -723,7 +724,7 @@ describe('AccountUsageCell', () => { user_cost: 0 }, grok_request_quota: { limit: 100, remaining: 100 }, - grok_token_quota: { limit: 2_000_000, remaining: 2_000_000 } + grok_token_quota: { limit: 1_000_000, remaining: 1_000_000 } }) const wrapper = mount(AccountUsageCell, { @@ -753,6 +754,7 @@ describe('AccountUsageCell', () => { it('Grok Free uses rolling 24h usage instead of today-only usage', async () => { getUsage.mockResolvedValue({ + grok_free_token_limit: 1_000_000, grok_billing: { period_type: 'weekly', usage_percent: null, plan: '' }, grok_local_usage: { requests: 2, @@ -762,7 +764,7 @@ describe('AccountUsageCell', () => { }, grok_local_usage_24h: { requests: 12, - tokens: 1_500_000, + tokens: 750_000, cost: 0, standard_cost: 0 } @@ -793,7 +795,7 @@ describe('AccountUsageCell', () => { await flushPromises() expect(wrapper.text()).toContain('24h|75|admin.accounts.usageWindow.grokFreeQuota24hHint') - expect(wrapper.text()).toContain('1.5M') + expect(wrapper.text()).toContain('750.0K') expect(wrapper.text()).not.toContain('7d|') expect(wrapper.text()).not.toContain('200.0K') expect(wrapper.text()).not.toContain('250.0K') @@ -801,6 +803,7 @@ describe('AccountUsageCell', () => { it('Grok Free does not substitute today stats when rolling 24h usage is unavailable', async () => { getUsage.mockResolvedValue({ + grok_free_token_limit: 1_000_000, grok_billing: { period_type: 'weekly', usage_percent: null, plan: '' }, grok_local_usage: { requests: 1, @@ -921,8 +924,9 @@ describe('AccountUsageCell', () => { expect(wrapper.text()).not.toContain('2M|') }) - it('Grok credential Free tier keeps the 2M fallback when billing is unavailable', async () => { + it('Grok credential Free tier keeps the 1M fallback when billing is unavailable', async () => { getUsage.mockResolvedValue({ + grok_free_token_limit: 1_000_000, subscription_tier: 'FREE', grok_local_usage_24h: { requests: 3, @@ -950,7 +954,7 @@ describe('AccountUsageCell', () => { await flushPromises() - expect(wrapper.text()).toContain('24h|50') + expect(wrapper.text()).toContain('24h|100') }) it('Grok paid manual probes keep the weekly/local summary when 24h usage is returned', async () => { @@ -1165,6 +1169,7 @@ describe('AccountUsageCell', () => { it('Grok Free manual probes merge rolling 24h usage', async () => { getUsage.mockResolvedValue({ + grok_free_token_limit: 1_000_000, subscription_tier: 'FREE', grok_quota_snapshot_state: 'no_headers' }) @@ -1185,7 +1190,7 @@ describe('AccountUsageCell', () => { template: ` + + + +
@@ -284,7 +359,12 @@ import { computed, onBeforeUnmount, onMounted, ref } from 'vue' import { useI18n } from 'vue-i18n' import { adminAPI } from '@/api' import { useAppStore } from '@/stores' -import type { BackupS3Config, BackupScheduleConfig, BackupRecord } from '@/api/admin/backup' +import type { + BackupS3Config, + BackupScheduleConfig, + BackupRecord, + ImageStorageConfig, +} from '@/api/admin/backup' import { useStepUp, isStepUpBlocked, isStepUpCancelled, stepUpBlockReason } from '@/composables/useStepUp' import TotpStepUpDialog from '@/components/auth/TotpStepUpDialog.vue' @@ -317,6 +397,26 @@ const s3SecretConfigured = ref(false) const savingS3 = ref(false) const testingS3 = ref(false) +// Async image object storage. Shares the S3 client with backups, so the default is +// to reuse the credentials configured above and only differ by prefix. +const imageStorageForm = ref({ + enabled: false, + reuse_backup_s3: true, + bucket: '', + prefix: 'images/', + public_base_url: '', + presign_expiry_hours: 24, + max_download_bytes: 33554432, + endpoint: '', + region: 'auto', + access_key_id: '', + secret_access_key: '', + force_path_style: false, +}) +const imageStorageSecretConfigured = ref(false) +const savingImageStorage = ref(false) +const testingImageStorage = ref(false) + // Schedule config const scheduleForm = ref({ enabled: false, @@ -485,6 +585,54 @@ async function saveS3Config() { } } +async function loadImageStorageConfig() { + try { + const { config, secret_configured } = await adminAPI.backup.getImageStorageConfig() + imageStorageForm.value = { + ...config, + prefix: config.prefix || 'images/', + region: config.region || 'auto', + secret_access_key: '', + } + imageStorageSecretConfigured.value = secret_configured + } catch (error) { + appStore.showError((error as { message?: string })?.message || t('errors.networkError')) + } +} + +async function saveImageStorageConfig() { + savingImageStorage.value = true + try { + await backupStepUp.run(() => adminAPI.backup.updateImageStorageConfig(imageStorageForm.value)) + appStore.showSuccess(t('admin.backup.imageStorage.saved')) + await loadImageStorageConfig() + } catch (error) { + if (isStepUpCancelled(error)) { + savingImageStorage.value = false + return + } + appStore.showError((error as { message?: string })?.message || t('errors.networkError')) + } finally { + savingImageStorage.value = false + } +} + +async function testImageStorage() { + testingImageStorage.value = true + try { + const result = await adminAPI.backup.testImageStorageConnection(imageStorageForm.value) + if (result.ok) { + appStore.showSuccess(result.message || t('admin.backup.s3.testSuccess')) + } else { + appStore.showError(result.message || t('admin.backup.s3.testFailed')) + } + } catch (error) { + appStore.showError((error as { message?: string })?.message || t('errors.networkError')) + } finally { + testingImageStorage.value = false + } +} + async function testS3() { testingS3.value = true try { @@ -642,7 +790,7 @@ function formatDate(value?: string): string { onMounted(async () => { document.addEventListener('visibilitychange', handleVisibilityChange) - await Promise.all([loadS3Config(), loadSchedule(), loadBackups()]) + await Promise.all([loadS3Config(), loadImageStorageConfig(), loadSchedule(), loadBackups()]) // 如果有正在 running 的备份，恢复轮询 const runningBackup = backups.value.find(r => r.status === 'running') From eea02c38fc9aa12ebf9b08609f0902da612e1670 Mon Sep 17 00:00:00 2001 From: superman2003 <2112076433zcr@gmail.com> Date: Sun, 19 Jul 2026 16:03:18 +0800 Subject: [PATCH 18/59] fix(grok): cache Claude Desktop client tools --- .../service/openai_gateway_grok_cache.go | 41 ++++- .../service/openai_gateway_grok_cache_test.go | 168 ++++++++++++++++++ .../service/openai_gateway_grok_test.go | 102 +++++++++++ 3 files changed, 308 insertions(+), 3 deletions(-) diff --git a/backend/internal/service/openai_gateway_grok_cache.go b/backend/internal/service/openai_gateway_grok_cache.go index 1ec6159c1a..ab81c16080 100644 --- a/backend/internal/service/openai_gateway_grok_cache.go +++ b/backend/internal/service/openai_gateway_grok_cache.go @@ -169,19 +169,54 @@ func applyGrokFreeMessagesFunctionToolCacheRoute(body, intentSourceBody []byte, // forwards the explicitly supported OpenAI-Beta header from downstream. func applyGrokFreeRequestToolCacheRoute(c *gin.Context, body, intentSourceBody []byte, account *Account, cacheIdentity string) ([]byte, error) { allowPureClientTools := account != nil && account.getExtraBool(grokClientToolCacheOptInExtraKey) - if !allowPureClientTools && c != nil { + requestOptOut := false + if c != nil { switch strings.ToLower(strings.TrimSpace(c.GetHeader(grokClientToolCacheOptInHeader))) { case "1", "true", "yes", "on", "prefer-cache": allowPureClientTools = true + case "0", "false", "no", "off": + allowPureClientTools = false + requestOptOut = true } } + if !allowPureClientTools && !requestOptOut && isGrokClaudeDesktopResponsesCacheRequest(c) { + allowPureClientTools = true + } // A function merely named web_search/x_search is still a client function. // Native Responses search types are safe to recognize automatically, while - // converting a client function requires the same explicit opt-in as every - // other pure client-tool request (#4486). + // converting a client function requires either explicit opt-in or the strict + // Claude Desktop Responses fingerprint used above (#4486). return applyGrokFreeToolCacheRoute(body, intentSourceBody, account, cacheIdentity, allowPureClientTools, allowPureClientTools) } +// isGrokClaudeDesktopResponsesCacheRequest recognizes the strict wire +// fingerprint emitted when Claude Desktop's local agent is translated by +// CC Switch into an OpenAI Responses request. Requiring every independent +// signal prevents a generic Claude-compatible client (or the Chat bridge) +// from silently opting into the mixed native/client tool route. +func isGrokClaudeDesktopResponsesCacheRequest(c *gin.Context) bool { + if c == nil || c.Request == nil || c.Request.URL == nil || isOpenAIResponsesCompactPath(c) { + return false + } + path := strings.TrimRight(strings.TrimSpace(c.Request.URL.Path), "/") + if !strings.HasSuffix(path, "/responses") { + return false + } + + if !claudeCodeUAPattern.MatchString(strings.TrimSpace(c.GetHeader("User-Agent"))) { + return false + } + switch strings.ToLower(strings.TrimSpace(c.GetHeader("X-App"))) { + case "cli", "cli-bg": + default: + return false + } + if !strings.EqualFold(strings.TrimSpace(c.GetHeader("anthropic-client-platform")), "desktop_app") { + return false + } + return strings.TrimSpace(c.GetHeader("X-Claude-Code-Session-Id")) != "" +} + func applyGrokFreeToolCacheRoute(body, intentSourceBody []byte, account *Account, cacheIdentity string, allowPureClientTools, allowFunctionSearch bool) ([]byte, error) { if strings.TrimSpace(cacheIdentity) == "" || !isKnownGrokFreeAccount(account) { return body, nil diff --git a/backend/internal/service/openai_gateway_grok_cache_test.go b/backend/internal/service/openai_gateway_grok_cache_test.go index a6f7d00881..d0d33ffb90 100644 --- a/backend/internal/service/openai_gateway_grok_cache_test.go +++ b/backend/internal/service/openai_gateway_grok_cache_test.go @@ -287,6 +287,174 @@ func TestGrokFreeClientToolCacheRequestOptIn(t *testing.T) { require.Equal(t, "x_search", tools[2].Get("type").String()) } +func TestGrokFreeClientToolCacheClaudeDesktopResponsesAutoOptIn(t *testing.T) { + account := healthyGrokOAuthGatewayTestAccount(90141, "access-token") + account.Credentials["subscription_tier"] = "free" + body := []byte(`{"model":"grok","tools":[{"type":"function","name":"Read","parameters":{"type":"object"}},{"type":"function","name":"Edit","parameters":{"type":"object"}}],"tool_choice":"auto"}`) + + for _, xApp := range []string{"cli", "cli-bg"} { + t.Run(xApp, func(t *testing.T) { + c := newGrokCacheTestContext(90141) + // The desktop marker text is intentionally not required; CC Switch and + // Claude Desktop may change the descriptive User-Agent suffix. + c.Request.Header.Set("User-Agent", "claude-cli/2.1.215 (external, future-desktop, agent-sdk/0.3.215)") + c.Request.Header.Set("X-App", xApp) + c.Request.Header.Set("anthropic-client-platform", "desktop_app") + c.Request.Header.Set("X-Claude-Code-Session-Id", "desktop-session-1") + + patched, err := applyGrokFreeRequestToolCacheRoute(c, body, body, account, "isolated-id") + + require.NoError(t, err) + tools := gjson.GetBytes(patched, "tools").Array() + require.Len(t, tools, 4) + require.Equal(t, "Read", tools[0].Get("name").String()) + require.Equal(t, "Edit", tools[1].Get("name").String()) + require.Equal(t, "web_search", tools[2].Get("type").String()) + require.Equal(t, "x_search", tools[3].Get("type").String()) + }) + } +} + +func TestGrokFreeClientToolCacheClaudeDesktopFingerprintRequiresAllSignals(t *testing.T) { + account := healthyGrokOAuthGatewayTestAccount(90142, "access-token") + account.Credentials["subscription_tier"] = "free" + body := []byte(`{"model":"grok","tools":[{"type":"function","name":"Read","parameters":{"type":"object"}}],"tool_choice":"auto"}`) + + tests := []struct { + name string + path string + ua string + xApp string + platform string + session string + }{ + { + name: "chat path", + path: "/v1/chat/completions", + ua: "claude-cli/2.1.215 (external, claude-desktop-3p, agent-sdk/0.3.215)", + xApp: "cli", + platform: "desktop_app", + session: "desktop-session-1", + }, + { + name: "compact responses path", + path: "/v1/responses/compact", + ua: "claude-cli/2.1.215 (external, claude-desktop-3p, agent-sdk/0.3.215)", + xApp: "cli", + platform: "desktop_app", + session: "desktop-session-1", + }, + { + name: "non claude cli user agent", + path: "/v1/responses", + ua: "Mozilla/5.0 (claude-desktop-3p)", + xApp: "cli", + platform: "desktop_app", + session: "desktop-session-1", + }, + { + name: "missing x app", + path: "/v1/responses", + ua: "claude-cli/2.1.215 (external, claude-desktop-3p, agent-sdk/0.3.215)", + platform: "desktop_app", + session: "desktop-session-1", + }, + { + name: "wrong x app", + path: "/v1/responses", + ua: "claude-cli/2.1.215 (external, claude-desktop-3p, agent-sdk/0.3.215)", + xApp: "desktop", + platform: "desktop_app", + session: "desktop-session-1", + }, + { + name: "missing client platform", + path: "/v1/responses", + ua: "claude-cli/2.1.215 (external, claude-desktop-3p, agent-sdk/0.3.215)", + xApp: "cli", + session: "desktop-session-1", + }, + { + name: "wrong client platform", + path: "/v1/responses", + ua: "claude-cli/2.1.215 (external, claude-desktop-3p, agent-sdk/0.3.215)", + xApp: "cli", + platform: "web", + session: "desktop-session-1", + }, + { + name: "missing session header", + path: "/v1/responses", + ua: "claude-cli/2.1.215 (external, claude-desktop-3p, agent-sdk/0.3.215)", + xApp: "cli", + platform: "desktop_app", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := newGrokCacheTestContext(90142) + c.Request.URL.Path = tt.path + if tt.ua != "" { + c.Request.Header.Set("User-Agent", tt.ua) + } + if tt.xApp != "" { + c.Request.Header.Set("X-App", tt.xApp) + } + if tt.platform != "" { + c.Request.Header.Set("anthropic-client-platform", tt.platform) + } + if tt.session != "" { + c.Request.Header.Set("X-Claude-Code-Session-Id", tt.session) + } + + patched, err := applyGrokFreeRequestToolCacheRoute(c, body, body, account, "isolated-id") + + require.NoError(t, err) + require.JSONEq(t, string(body), string(patched)) + }) + } +} + +func TestGrokFreeClientToolCacheClaudeDesktopExplicitRequestOptOut(t *testing.T) { + account := healthyGrokOAuthGatewayTestAccount(90144, "access-token") + account.Credentials["subscription_tier"] = "free" + account.Extra = map[string]any{grokClientToolCacheOptInExtraKey: true} + body := []byte(`{"model":"grok","tools":[{"type":"function","name":"Read","parameters":{"type":"object"}}],"tool_choice":"auto"}`) + + for _, value := range []string{"0", "false", "no", "off"} { + t.Run(value, func(t *testing.T) { + c := newGrokCacheTestContext(90144) + c.Request.Header.Set("User-Agent", "claude-cli/2.1.215 (external, claude-desktop-3p, agent-sdk/0.3.215)") + c.Request.Header.Set("X-App", "cli") + c.Request.Header.Set("anthropic-client-platform", "desktop_app") + c.Request.Header.Set("X-Claude-Code-Session-Id", "desktop-session-1") + c.Request.Header.Set(grokClientToolCacheOptInHeader, value) + + patched, err := applyGrokFreeRequestToolCacheRoute(c, body, body, account, "isolated-id") + + require.NoError(t, err) + require.JSONEq(t, string(body), string(patched)) + }) + } +} + +func TestGrokFreeClientToolCacheClaudeDesktopAutoOptInDoesNotOverridePaidTier(t *testing.T) { + account := healthyGrokOAuthGatewayTestAccount(90143, "access-token") + account.Credentials["subscription_tier"] = "supergrok" + c := newGrokCacheTestContext(90143) + c.Request.Header.Set("User-Agent", "claude-cli/2.1.215 (external, claude-desktop-3p, agent-sdk/0.3.215)") + c.Request.Header.Set("X-App", "cli") + c.Request.Header.Set("anthropic-client-platform", "desktop_app") + c.Request.Header.Set("X-Claude-Code-Session-Id", "desktop-session-1") + body := []byte(`{"model":"grok","tools":[{"type":"function","name":"Read","parameters":{"type":"object"}}],"tool_choice":"auto"}`) + + patched, err := applyGrokFreeRequestToolCacheRoute(c, body, body, account, "isolated-id") + + require.NoError(t, err) + require.JSONEq(t, string(body), string(patched)) +} + func TestGrokFreeRequestClientSearchFunctionRequiresOptIn(t *testing.T) { account := healthyGrokOAuthGatewayTestAccount(9015, "access-token") account.Credentials["subscription_tier"] = "free" diff --git a/backend/internal/service/openai_gateway_grok_test.go b/backend/internal/service/openai_gateway_grok_test.go index 42d88875db..bffb2744c5 100644 --- a/backend/internal/service/openai_gateway_grok_test.go +++ b/backend/internal/service/openai_gateway_grok_test.go @@ -329,6 +329,108 @@ func TestForwardGrokResponsesCodexAdditionalToolsUsesMixedCacheIntent(t *testing require.Empty(t, upstream.lastReq.Header.Get(grokClientToolCacheOptInHeader)) } +func TestForwardGrokResponsesClaudeDesktopClientToolsUseCacheRoute(t *testing.T) { + gin.SetMode(gin.TestMode) + + firstBody := []byte(`{ + "model":"grok","stream":false,"instructions":"You are Claude Desktop.", + "tools":[ + {"type":"function","name":"Read","parameters":{"type":"object"}}, + {"type":"function","name":"Edit","parameters":{"type":"object"}}, + {"type":"function","name":"WebSearch","parameters":{"type":"object"}}, + {"type":"function","name":"mcp__workspace__bash","parameters":{"type":"object"}} + ], + "input":[{"role":"user","content":[{"type":"input_text","text":"first turn"}]}] + }`) + secondBody := []byte(`{ + "model":"grok","stream":false,"instructions":"You are Claude Desktop.", + "tools":[ + {"type":"function","name":"Read","parameters":{"type":"object"}}, + {"type":"function","name":"Edit","parameters":{"type":"object"}}, + {"type":"function","name":"WebSearch","parameters":{"type":"object"}}, + {"type":"function","name":"mcp__workspace__bash","parameters":{"type":"object"}} + ], + "input":[ + {"role":"user","content":[{"type":"input_text","text":"first turn"}]}, + {"role":"assistant","content":[{"type":"output_text","text":"first answer"}]}, + {"role":"user","content":[{"type":"input_text","text":"second turn"}]} + ] + }`) + + account := healthyGrokOAuthGatewayTestAccount(4504, "access-token") + account.Credentials["subscription_tier"] = "free" + repo := &grokQuotaAccountRepo{ + mockAccountRepoForPlatform: &mockAccountRepoForPlatform{ + accountsByID: map[int64]*Account{account.ID: account}, + }, + } + upstream := &httpUpstreamRecorder{responses: []*http.Response{ + { + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{ + "id":"resp_claude_desktop_1","object":"response","model":"grok-4.5","status":"completed", + "output":[],"usage":{"input_tokens":30000,"output_tokens":10,"input_tokens_details":{"cached_tokens":0}} + }`)), + }, + { + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{ + "id":"resp_claude_desktop_2","object":"response","model":"grok-4.5","status":"completed", + "output":[],"usage":{"input_tokens":30100,"output_tokens":12,"input_tokens_details":{"cached_tokens":28672}} + }`)), + }, + }} + svc := &OpenAIGatewayService{ + httpUpstream: upstream, + grokTokenProvider: NewGrokTokenProvider(repo, nil), + accountRepo: repo, + } + + newContext := func(body []byte) *gin.Context { + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(body)) + c.Request.Header.Set("User-Agent", "claude-cli/2.1.215 (external, claude-desktop-3p, agent-sdk/0.3.215)") + c.Request.Header.Set("X-App", "cli") + c.Request.Header.Set("anthropic-client-platform", "desktop_app") + c.Request.Header.Set("X-Claude-Code-Session-Id", "claude-desktop-session") + c.Set("api_key", &APIKey{ID: 4504}) + return c + } + + first, err := svc.forwardGrokResponses(context.Background(), newContext(firstBody), account, firstBody, "grok", false, time.Now()) + require.NoError(t, err) + second, err := svc.forwardGrokResponses(context.Background(), newContext(secondBody), account, secondBody, "grok", false, time.Now()) + require.NoError(t, err) + + require.Equal(t, 0, first.Usage.CacheReadInputTokens) + require.Equal(t, 28672, second.Usage.CacheReadInputTokens) + require.Len(t, upstream.bodies, 2) + require.Len(t, upstream.requests, 2) + for i := range upstream.bodies { + tools := gjson.GetBytes(upstream.bodies[i], "tools").Array() + require.Len(t, tools, 6) + require.Equal(t, "Read", tools[0].Get("name").String()) + require.Equal(t, "Edit", tools[1].Get("name").String()) + require.Equal(t, "WebSearch", tools[2].Get("name").String()) + require.Equal(t, "mcp__workspace__bash", tools[3].Get("name").String()) + require.Equal(t, "web_search", tools[4].Get("type").String()) + require.Equal(t, "x_search", tools[5].Get("type").String()) + require.False(t, gjson.GetBytes(upstream.bodies[i], "tool_choice").Exists()) + require.Empty(t, upstream.requests[i].Header.Get("X-App")) + require.Empty(t, upstream.requests[i].Header.Get("anthropic-client-platform")) + require.Empty(t, upstream.requests[i].Header.Get("X-Claude-Code-Session-Id")) + } + firstIdentity := gjson.GetBytes(upstream.bodies[0], "prompt_cache_key").String() + secondIdentity := gjson.GetBytes(upstream.bodies[1], "prompt_cache_key").String() + require.NotEmpty(t, firstIdentity) + require.Equal(t, firstIdentity, secondIdentity) + require.Equal(t, firstIdentity, upstream.requests[0].Header.Get(grokConversationIDHeader)) + require.Equal(t, secondIdentity, upstream.requests[1].Header.Get(grokConversationIDHeader)) +} + func TestGrokResponsesCacheIdentityIncludesPromotedCodexTools(t *testing.T) { c := newGrokCacheTestContext(4503) lookupBody := []byte(`{"model":"grok","input":[{"type":"additional_tools","tools":[{"type":"function","name":"lookup","parameters":{"type":"object"}}]},{"type":"message","role":"user","content":"same prompt"}]}`) From 6102d64274fa002df81803f2b840d3ac9659a39c Mon Sep 17 00:00:00 2001 From: haruka <1628615876@qq.com> Date: Sun, 19 Jul 2026 01:03:41 -0700 Subject: [PATCH 19/59] =?UTF-8?q?fix(wire):=20=E8=A1=A5=E4=B8=8A=20PromptA?= =?UTF-8?q?dminService=20=E7=9A=84=E7=BB=91=E5=AE=9A=EF=BC=8C=E6=81=A2?= =?UTF-8?q?=E5=A4=8D=20wire=20=E4=BB=A3=E7=A0=81=E7=94=9F=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ProviderSet 提供了 NewPromptAdminHandler，也绑定了 PromptEngine，却没有为 PromptAdminService 绑定实现，导致 go generate ./cmd/server 直接失败： no provider found for securityaudit.PromptAdminService needed by *securityaudit.PromptAdminHandler *PromptService 本就实现了该接口的全部方法，wire_gen.go 也一直是手工维护到位的 （补上绑定后重新生成的结果与现有文件逐字节一致），只是生成这条路被堵死了， 后续任何依赖改动都无法再靠 go generate 同步。 Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01VHreE5pzCkSYz7J45fmd2Y --- backend/internal/securityaudit/prompt_module.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/internal/securityaudit/prompt_module.go b/backend/internal/securityaudit/prompt_module.go index 691cc7dbdb..e1199fcfaf 100644 --- a/backend/internal/securityaudit/prompt_module.go +++ b/backend/internal/securityaudit/prompt_module.go @@ -16,6 +16,7 @@ var ProviderSet = wire.NewSet( wire.Bind(new(ConfigStore), new(*ConfigManager)), NewPromptService, wire.Bind(new(PromptEngine), new(*PromptService)), + wire.Bind(new(PromptAdminService), new(*PromptService)), NewLegacyModerationAdapter, NewCoordinator, NewPromptAdminHandler, From 7840eb1c49384a176ab7a1f68f9c056b46a9d895 Mon Sep 17 00:00:00 2001 From: Kinso Date: Sun, 19 Jul 2026 17:13:27 +0800 Subject: [PATCH 20/59] [verified] fix(openai): make HTTP bridge failover safe Route first-turn pre-response transport, HTTP, and SSE failures through the existing account failover path without replaying later turns. Require a real Responses terminal event and preserve platform-specific OpenAI/Grok error side effects. --- .../service/openai_gateway_grok_test.go | 49 ++++ .../internal/service/openai_ws_http_bridge.go | 74 ++++-- .../service/openai_ws_http_bridge_test.go | 213 ++++++++++++++++++ 3 files changed, 317 insertions(+), 19 deletions(-) diff --git a/backend/internal/service/openai_gateway_grok_test.go b/backend/internal/service/openai_gateway_grok_test.go index fd7eb09c3d..1709e2485f 100644 --- a/backend/internal/service/openai_gateway_grok_test.go +++ b/backend/internal/service/openai_gateway_grok_test.go @@ -16,6 +16,7 @@ import ( "testing" "time" + "github.com/Wei-Shaw/sub2api/internal/config" "github.com/Wei-Shaw/sub2api/internal/pkg/xai" "github.com/gin-gonic/gin" "github.com/stretchr/testify/require" @@ -2498,6 +2499,54 @@ func TestOpenAIWSHTTPBridgeGrok429PersistsRateLimit(t *testing.T) { require.True(t, svc.isOpenAIAccountRuntimeBlocked(account)) } +func TestOpenAIWSHTTPBridgeSSEErrorSideEffectsRunOncePerPlatform(t *testing.T) { + gin.SetMode(gin.TestMode) + + for _, platform := range []string{PlatformOpenAI, PlatformGrok} { + t.Run(platform, func(t *testing.T) { + repo := &grokQuotaAccountRepo{} + cfg := &config.Config{} + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader( + "data: {\"type\":\"error\",\"error\":{\"type\":\"rate_limit_error\",\"code\":\"rate_limit_exceeded\",\"message\":\"limited\"}}\n\n", + )), + }} + svc := &OpenAIGatewayService{ + cfg: cfg, + accountRepo: repo, + httpUpstream: upstream, + } + if platform == PlatformOpenAI { + svc.rateLimitService = NewRateLimitService(repo, nil, cfg, nil, nil) + } + account := &Account{ID: 70, Platform: platform, Type: AccountTypeOAuth, Concurrency: 1} + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/v1/responses", nil) + payload := []byte(`{"type":"response.create","model":"gpt-5","input":"hi"}`) + writes := 0 + + result, err := svc.proxyOpenAIWSHTTPBridgeTurn( + context.Background(), c, account, "sk-test", payload, len(payload), + "gpt-5", "", "", "", "", 1, + func([]byte) error { + writes++ + return nil + }, + ) + + require.Nil(t, result) + var failoverErr *UpstreamFailoverError + require.ErrorAs(t, err, &failoverErr) + require.Equal(t, http.StatusTooManyRequests, failoverErr.StatusCode) + require.Zero(t, writes) + require.Equal(t, 1, repo.rateLimitedCalls) + }) + } +} + func TestOpenAIWSHTTPBridgeGrokExhaustedSuccessPersistsRateLimit(t *testing.T) { repo := &grokQuotaAccountRepo{} resetAt := time.Now().Add(20 * time.Minute).UTC().Truncate(time.Second) diff --git a/backend/internal/service/openai_ws_http_bridge.go b/backend/internal/service/openai_ws_http_bridge.go index cc7b307982..a84968061b 100644 --- a/backend/internal/service/openai_ws_http_bridge.go +++ b/backend/internal/service/openai_ws_http_bridge.go @@ -230,6 +230,9 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( turnStart := time.Now() resp, err := s.httpUpstream.Do(upstreamReq, proxyURL, account.ID, account.Concurrency) if err != nil { + if turn == 1 { + return nil, s.handleOpenAIUpstreamTransportError(ctx, c, account, err, true) + } safeErr := sanitizeUpstreamErrorMessage(err.Error()) _ = writeClientMessage(buildOpenAIWSHTTPBridgeErrorEvent(http.StatusBadGateway, "Upstream request failed")) return nil, fmt.Errorf("upstream http bridge request failed: %s", safeErr) @@ -238,16 +241,23 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( if resp.StatusCode >= 400 { respBody, _ := io.ReadAll(io.LimitReader(resp.Body, openAIWSHTTPBridgeErrorBodyLimitBytes)) - if account.Platform == PlatformGrok { - s.handleGrokAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, respBody) - } else if shouldCooldownOpenAITransientUpstreamError(resp.StatusCode, respBody) { - canonicalModel := canonicalOpenAIAccountSchedulingModel(account, originalModel) - s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, respBody, canonicalModel) - } upstreamMsg := sanitizeUpstreamErrorMessage(strings.TrimSpace(extractUpstreamErrorMessage(respBody))) if upstreamMsg == "" { upstreamMsg = http.StatusText(resp.StatusCode) } + shouldFailover := s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMsg, respBody) + if account.Platform == PlatformGrok { + s.handleGrokAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, respBody) + if turn == 1 && shouldFailover { + return nil, newOpenAIUpstreamFailoverError(resp.StatusCode, resp.Header, respBody, upstreamMsg, false) + } + } else if turn == 1 && shouldFailover { + return nil, s.handleFailoverErrorResponsePassthrough(ctx, resp, c, account, body, respBody) + } + if account.Platform != PlatformGrok && (shouldFailover || shouldCooldownOpenAITransientUpstreamError(resp.StatusCode, respBody)) { + canonicalModel := canonicalOpenAIAccountSchedulingModel(account, originalModel) + s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, respBody, canonicalModel) + } _ = writeClientMessage(buildOpenAIWSHTTPBridgeErrorEvent(resp.StatusCode, upstreamMsg)) return nil, fmt.Errorf("upstream http bridge error: status=%d message=%s", resp.StatusCode, upstreamMsg) } @@ -372,6 +382,31 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( } replayCollector.AddEvent(eventType, upstreamMessage) + var upstreamEventErr error + if eventType == "error" { + errCodeRaw, errTypeRaw, errMsgRaw := parseOpenAIWSErrorEventFields(upstreamMessage) + errMessage := strings.TrimSpace(errMsgRaw) + if errMessage == "" { + errMessage = "upstream error event" + } + statusCode := openAIWSErrorHTTPStatusFromRaw(errCodeRaw, errTypeRaw) + shouldFailover := s.shouldFailoverOpenAIUpstreamResponse(statusCode, errMessage, upstreamMessage) + if account.Platform == PlatformGrok { + s.handleGrokAccountUpstreamError(ctx, account, statusCode, resp.Header, upstreamMessage) + } else if shouldFailover { + accountStatus := statusCode + if transientStatus := openAIWSPayloadTransientStatus(upstreamMessage); transientStatus != 0 { + accountStatus = transientStatus + } + canonicalModel := canonicalOpenAIAccountSchedulingModel(account, originalModel) + s.handleOpenAIAccountUpstreamError(ctx, account, accountStatus, resp.Header, upstreamMessage, canonicalModel) + } + if turn == 1 && !wroteDownstream && shouldFailover { + return nil, newOpenAIUpstreamFailoverError(statusCode, resp.Header, upstreamMessage, errMessage, false) + } + upstreamEventErr = errors.New(errMessage) + } + if !clientDisconnected { if err := writeClientMessage(upstreamMessage); err != nil { if isOpenAIWSClientDisconnectError(err) { @@ -396,15 +431,8 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( } } - if eventType == "error" { - s.handleOpenAIWSErrorEventTransientFailure(ctx, account, canonicalOpenAIAccountSchedulingModel(account, originalModel), resp.Header, upstreamMessage) - errCodeRaw, errTypeRaw, errMsgRaw := parseOpenAIWSErrorEventFields(upstreamMessage) - s.persistOpenAIWSRateLimitSignal(ctx, account, resp.Header, upstreamMessage, errCodeRaw, errTypeRaw, errMsgRaw) - errMessage := strings.TrimSpace(errMsgRaw) - if errMessage == "" { - errMessage = "upstream error event" - } - return resultWithUsage(), errors.New(errMessage) + if upstreamEventErr != nil { + return resultWithUsage(), upstreamEventErr } if isOpenAIWSTerminalEvent(eventType) { upstreamTerminalEvent = s.handleOpenAIWSTerminalTransientFailure(ctx, account, canonicalOpenAIAccountSchedulingModel(account, originalModel), resp.Header, upstreamMessage) @@ -432,12 +460,20 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( } } if err := scanner.Err(); err != nil { - return resultWithUsage(), fmt.Errorf("read upstream http bridge stream: %w", err) + streamErr := fmt.Errorf("read upstream http bridge stream: %w", err) + if turn == 1 && !wroteDownstream { + return nil, s.handleOpenAIUpstreamTransportError(ctx, c, account, streamErr, true) + } + return resultWithUsage(), streamErr } - if sawDone && eventCount > 0 { - return resultWithUsage(), nil + terminalErr := errors.New("upstream http bridge stream ended before terminal event") + if sawDone { + terminalErr = errors.New("upstream http bridge stream sent [DONE] before terminal event") } - return resultWithUsage(), errors.New("upstream http bridge stream ended before terminal event") + if turn == 1 && !wroteDownstream { + return nil, s.handleOpenAIUpstreamTransportError(ctx, c, account, terminalErr, true) + } + return resultWithUsage(), terminalErr } func resolveGrokWSCacheIdentity(c *gin.Context, account *Account, payload []byte, originalModel string) (string, error) { diff --git a/backend/internal/service/openai_ws_http_bridge_test.go b/backend/internal/service/openai_ws_http_bridge_test.go index 808a63c6b1..de2e463816 100644 --- a/backend/internal/service/openai_ws_http_bridge_test.go +++ b/backend/internal/service/openai_ws_http_bridge_test.go @@ -62,6 +62,219 @@ func TestOpenAIWSHTTPBridgeDecisionKeepsSmallFramesOnWS(t *testing.T) { require.True(t, svc.shouldBridgeOpenAIWSHTTP(&Account{Platform: PlatformGrok}, 1, "resp_existing")) } +func TestProxyOpenAIWSHTTPBridgeTurnTransportErrorFailoverSafety(t *testing.T) { + gin.SetMode(gin.TestMode) + + tests := []struct { + name string + turn int + wantFailover bool + wantWrites int + }{ + {name: "first_turn_fails_over_before_downstream_event", turn: 1, wantFailover: true}, + {name: "later_turn_does_not_replay_completed_turns", turn: 2, wantWrites: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream := &httpUpstreamRecorder{err: io.EOF} + svc := &OpenAIGatewayService{ + cfg: &config.Config{}, + httpUpstream: upstream, + } + account := &Account{ + ID: 8, + Name: "api-key", + Platform: PlatformOpenAI, + Type: AccountTypeAPIKey, + Concurrency: 1, + } + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/v1/responses", nil) + payload := []byte(`{"type":"response.create","model":"gpt-5","input":"hi"}`) + var writes [][]byte + + result, err := svc.proxyOpenAIWSHTTPBridgeTurn( + context.Background(), c, account, "sk-test", payload, len(payload), + "gpt-5", "", "", "", "", tt.turn, + func(message []byte) error { + writes = append(writes, append([]byte(nil), message...)) + return nil + }, + ) + + require.Nil(t, result) + var failoverErr *UpstreamFailoverError + if tt.wantFailover { + require.ErrorAs(t, err, &failoverErr) + require.Equal(t, http.StatusBadGateway, failoverErr.StatusCode) + require.JSONEq(t, string(openAITransportFailoverBody), string(failoverErr.ResponseBody)) + } else { + require.Error(t, err) + require.False(t, errors.As(err, &failoverErr)) + } + require.Len(t, writes, tt.wantWrites) + if tt.wantWrites > 0 { + require.Equal(t, "error", gjson.GetBytes(writes[0], "type").String()) + require.Equal(t, int64(http.StatusBadGateway), gjson.GetBytes(writes[0], "status").Int()) + } + }) + } +} + +func TestProxyOpenAIWSHTTPBridgeTurnHTTPStatusFailoverSafety(t *testing.T) { + gin.SetMode(gin.TestMode) + + tests := []struct { + name string + turn int + status int + wantFailover bool + wantWrites int + }{ + {name: "first_turn_401", turn: 1, status: http.StatusUnauthorized, wantFailover: true}, + {name: "first_turn_429", turn: 1, status: http.StatusTooManyRequests, wantFailover: true}, + {name: "first_turn_500", turn: 1, status: http.StatusInternalServerError, wantFailover: true}, + {name: "later_turn_500_does_not_replay", turn: 2, status: http.StatusInternalServerError, wantWrites: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: tt.status, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"error":{"type":"server_error","message":"temporary upstream failure"}}`)), + }} + svc := &OpenAIGatewayService{cfg: &config.Config{}, httpUpstream: upstream} + account := &Account{ID: 9, Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Concurrency: 1} + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/v1/responses", nil) + payload := []byte(`{"type":"response.create","model":"gpt-5","input":"hi"}`) + var writes [][]byte + + result, err := svc.proxyOpenAIWSHTTPBridgeTurn( + context.Background(), c, account, "sk-test", payload, len(payload), + "gpt-5", "", "", "", "", tt.turn, + func(message []byte) error { + writes = append(writes, append([]byte(nil), message...)) + return nil + }, + ) + + require.Nil(t, result) + var failoverErr *UpstreamFailoverError + if tt.wantFailover { + require.ErrorAs(t, err, &failoverErr) + require.Equal(t, tt.status, failoverErr.StatusCode) + } else { + require.Error(t, err) + require.False(t, errors.As(err, &failoverErr)) + } + require.Len(t, writes, tt.wantWrites) + }) + } +} + +func TestProxyOpenAIWSHTTPBridgeTurnSSEErrorFailoverSafety(t *testing.T) { + gin.SetMode(gin.TestMode) + + for _, turn := range []int{1, 2} { + t.Run(fmt.Sprintf("turn_%d", turn), func(t *testing.T) { + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader( + "data: {\"type\":\"error\",\"error\":{\"type\":\"rate_limit_error\",\"code\":\"rate_limit_exceeded\",\"message\":\"limited\"}}\n\n", + )), + }} + svc := &OpenAIGatewayService{cfg: &config.Config{}, httpUpstream: upstream} + account := &Account{ID: 10, Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Concurrency: 1} + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/v1/responses", nil) + payload := []byte(`{"type":"response.create","model":"gpt-5","input":"hi"}`) + var writes [][]byte + + result, err := svc.proxyOpenAIWSHTTPBridgeTurn( + context.Background(), c, account, "sk-test", payload, len(payload), + "gpt-5", "", "", "", "", turn, + func(message []byte) error { + writes = append(writes, append([]byte(nil), message...)) + return nil + }, + ) + + var failoverErr *UpstreamFailoverError + if turn == 1 { + require.Nil(t, result) + require.ErrorAs(t, err, &failoverErr) + require.Equal(t, http.StatusTooManyRequests, failoverErr.StatusCode) + require.Empty(t, writes) + } else { + require.NotNil(t, result) + require.Error(t, err) + require.False(t, errors.As(err, &failoverErr)) + require.Len(t, writes, 1) + } + }) + } +} + +func TestProxyOpenAIWSHTTPBridgeTurnRequiresTerminalEvent(t *testing.T) { + gin.SetMode(gin.TestMode) + + tests := []struct { + name string + body string + wantFailover bool + wantWrites int + }{ + {name: "done_without_events_fails_over", body: "data: [DONE]\n\n", wantFailover: true}, + { + name: "created_then_done_is_truncated_not_success", + body: "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_truncated\"}}\n\n" + + "data: [DONE]\n\n", + wantWrites: 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(tt.body)), + }} + svc := &OpenAIGatewayService{cfg: &config.Config{}, httpUpstream: upstream} + account := &Account{ID: 11, Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Concurrency: 1} + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/v1/responses", nil) + payload := []byte(`{"type":"response.create","model":"gpt-5","input":"hi"}`) + var writes [][]byte + + result, err := svc.proxyOpenAIWSHTTPBridgeTurn( + context.Background(), c, account, "sk-test", payload, len(payload), + "gpt-5", "", "", "", "", 1, + func(message []byte) error { + writes = append(writes, append([]byte(nil), message...)) + return nil + }, + ) + + var failoverErr *UpstreamFailoverError + if tt.wantFailover { + require.Nil(t, result) + require.ErrorAs(t, err, &failoverErr) + } else { + require.NotNil(t, result) + require.Error(t, err) + require.False(t, errors.As(err, &failoverErr)) + } + require.Len(t, writes, tt.wantWrites) + }) + } +} + func TestOpenAIWSHTTPBridgeRelaysSSEFramesAsWebSocketMessages(t *testing.T) { gin.SetMode(gin.TestMode) From 8fd6f91b7e49030f46a859e6518733ff86175908 Mon Sep 17 00:00:00 2001 From: jjaw Date: Sun, 19 Jul 2026 18:46:20 +0800 Subject: [PATCH 21/59] =?UTF-8?q?=E4=BC=98=E5=8C=96=20Responses=20?= =?UTF-8?q?=E7=94=9F=E5=9B=BE=E6=84=8F=E5=9B=BE=E5=A4=8D=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../handler/openai_gateway_handler.go | 16 +++--- .../handler/openai_gateway_handler_test.go | 33 ++++++++++++ ...i_responses_image_intent_benchmark_test.go | 53 +++++++++++++++++++ 3 files changed, 96 insertions(+), 6 deletions(-) create mode 100644 backend/internal/handler/openai_responses_image_intent_benchmark_test.go diff --git a/backend/internal/handler/openai_gateway_handler.go b/backend/internal/handler/openai_gateway_handler.go index 602b971f44..7b697d6d69 100644 --- a/backend/internal/handler/openai_gateway_handler.go +++ b/backend/internal/handler/openai_gateway_handler.go @@ -124,6 +124,13 @@ func openAICompatibleRequestPlatform(apiKey *service.APIKey) string { return service.PlatformOpenAI } +func openAIResponsesRequiredCapability(imageIntent bool, platform string) service.OpenAIEndpointCapability { + if imageIntent && platform == service.PlatformOpenAI { + return service.OpenAIEndpointCapabilityResponses + } + return service.OpenAIEndpointCapabilityChatCompletions +} + func allowOpenAICompatibleMessagesDispatch(apiKey *service.APIKey) bool { if apiKey == nil || apiKey.Group == nil { return true @@ -363,12 +370,9 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { // 生图意图的 /v1/responses 请求必须调度到确实支持 Responses API 的账号，否则 // 会在 forward 阶段被静默降级为无法生图的 Chat Completions 直转（#4417）。 // 仅对 OpenAI 平台生效：Grok 生图走独立的 forwardGrokResponses 路径，不应被过滤。 - // 使用 IsExplicitImageGenerationIntent 排除被动 image_gen namespace 声明， - // 避免 Codex 的被动工具目录使 CC-only 账号被误过滤（#4476）。 - requiredCapability := service.OpenAIEndpointCapabilityChatCompletions - if service.IsExplicitImageGenerationIntent("/v1/responses", reqModel, body) && requestPlatform == service.PlatformOpenAI { - requiredCapability = service.OpenAIEndpointCapabilityResponses - } + // 复用前置权限与并发阶段在未修改 body 上确认的显式生图意图，避免大 tools 请求重复扫描。 + // 该判断已排除 Codex 被动 image_gen namespace，避免 CC-only 账号被误过滤（#4476）。 + requiredCapability := openAIResponsesRequiredCapability(imageIntent, requestPlatform) for { // Streaming Forward intentionally detaches the upstream request so usage can diff --git a/backend/internal/handler/openai_gateway_handler_test.go b/backend/internal/handler/openai_gateway_handler_test.go index 635c22c3f8..eace1c7c92 100644 --- a/backend/internal/handler/openai_gateway_handler_test.go +++ b/backend/internal/handler/openai_gateway_handler_test.go @@ -138,6 +138,39 @@ func TestOpenAIForwardSucceededForScheduling(t *testing.T) { })) } +func TestOpenAIResponsesRequiredCapability(t *testing.T) { + tests := []struct { + name string + imageIntent bool + platform string + want service.OpenAIEndpointCapability + }{ + { + name: "OpenAI explicit image intent requires Responses", + imageIntent: true, + platform: service.PlatformOpenAI, + want: service.OpenAIEndpointCapabilityResponses, + }, + { + name: "Grok explicit image intent keeps chat capability", + imageIntent: true, + platform: service.PlatformGrok, + want: service.OpenAIEndpointCapabilityChatCompletions, + }, + { + name: "non-image intent keeps chat capability", + platform: service.PlatformOpenAI, + want: service.OpenAIEndpointCapabilityChatCompletions, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, openAIResponsesRequiredCapability(tt.imageIntent, tt.platform)) + }) + } +} + func TestResolveOpenAIMessagesMetadataSession_DoesNotDerivePromptCacheKey(t *testing.T) { body := []byte(`{"model":"claude-sonnet-4-5","metadata":{"user_id":"claude-code-session"},"messages":[{"role":"user","content":"hello"}]}`) diff --git a/backend/internal/handler/openai_responses_image_intent_benchmark_test.go b/backend/internal/handler/openai_responses_image_intent_benchmark_test.go new file mode 100644 index 0000000000..cb6360e09e --- /dev/null +++ b/backend/internal/handler/openai_responses_image_intent_benchmark_test.go @@ -0,0 +1,53 @@ +package handler + +import ( + "strings" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/service" +) + +var openAIResponsesImageIntentRoutingBenchmarkSink service.OpenAIEndpointCapability + +func BenchmarkOpenAIResponsesImageIntentRouting_LargeToolsBody(b *testing.B) { + body := buildLargeOpenAIResponsesToolsBody(32 << 20) + if service.IsExplicitImageGenerationIntent("/v1/responses", "gpt-5.4", body) { + b.Fatal("large tools body must not have explicit image intent") + } + platform := service.PlatformOpenAI + + b.Run("reuse_once", func(b *testing.B) { + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for range b.N { + imageIntent := service.IsExplicitImageGenerationIntent("/v1/responses", "gpt-5.4", body) + openAIResponsesImageIntentRoutingBenchmarkSink = openAIResponsesRequiredCapability(imageIntent, platform) + } + }) + + b.Run("rescan_twice", func(b *testing.B) { + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for range b.N { + imageIntent := service.IsExplicitImageGenerationIntent("/v1/responses", "gpt-5.4", body) + // 对照优化前路径：路由阶段会再次扫描同一份未修改的 body。 + requiredCapability := service.OpenAIEndpointCapabilityChatCompletions + if service.IsExplicitImageGenerationIntent("/v1/responses", "gpt-5.4", body) && platform == service.PlatformOpenAI { + requiredCapability = service.OpenAIEndpointCapabilityResponses + } + if imageIntent && requiredCapability != service.OpenAIEndpointCapabilityResponses { + b.Fatal("explicit image intent must require Responses") + } + openAIResponsesImageIntentRoutingBenchmarkSink = requiredCapability + } + }) +} + +func buildLargeOpenAIResponsesToolsBody(targetBytes int) []byte { + var builder strings.Builder + builder.Grow(targetBytes + 256) + _, _ = builder.WriteString(`{"model":"gpt-5.4","tools":[{"type":"function","name":"search","description":"`) + _, _ = builder.WriteString(strings.Repeat("x", targetBytes)) + _, _ = builder.WriteString(`"}],"tool_choice":"auto","input":"write code"}`) + return []byte(builder.String()) +} From f7979ae8ff3f20a533e8984d7b8841d45c7fee58 Mon Sep 17 00:00:00 2001 From: jjaw Date: Sun, 19 Jul 2026 18:49:11 +0800 Subject: [PATCH 22/59] =?UTF-8?q?=E4=BC=98=E5=8C=96=20Responses=20SSE=20?= =?UTF-8?q?=E4=BA=8B=E4=BB=B6=E7=B1=BB=E5=9E=8B=E8=A7=A3=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/internal/service/gateway_service.go | 18 ++++- .../openai_gateway_response_flush_test.go | 33 +++++++++ .../openai_gateway_response_handling.go | 6 +- ...nai_gateway_response_handling_type_test.go | 74 +++++++++++++++++++ 4 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 backend/internal/service/openai_gateway_response_handling_type_test.go diff --git a/backend/internal/service/gateway_service.go b/backend/internal/service/gateway_service.go index 73c59c4ece..23684af4aa 100644 --- a/backend/internal/service/gateway_service.go +++ b/backend/internal/service/gateway_service.go @@ -173,7 +173,23 @@ func openAIStreamEventIsTerminal(data string) bool { if trimmed == "[DONE]" { return true } - switch gjson.Get(trimmed, "type").String() { + return openAIStreamEventTypeIsTerminal(gjson.Get(trimmed, "type").String()) +} + +// openAIStreamEventIsTerminalWithType 复用已提取的 type，避免 SSE 热路径重复扫描 JSON。 +func openAIStreamEventIsTerminalWithType(data, eventType string) bool { + trimmed := strings.TrimSpace(data) + if trimmed == "" { + return false + } + if trimmed == "[DONE]" { + return true + } + return openAIStreamEventTypeIsTerminal(eventType) +} + +func openAIStreamEventTypeIsTerminal(eventType string) bool { + switch eventType { case "response.completed", "response.done", "response.failed", "response.incomplete", "response.cancelled", "response.canceled": return true default: diff --git a/backend/internal/service/openai_gateway_response_flush_test.go b/backend/internal/service/openai_gateway_response_flush_test.go index 3494824d6c..2c35374596 100644 --- a/backend/internal/service/openai_gateway_response_flush_test.go +++ b/backend/internal/service/openai_gateway_response_flush_test.go @@ -402,6 +402,39 @@ func TestOpenAIResponseFlush_FailedAndErrorEventsFlushAtBoundaries(t *testing.T) }) } +func TestOpenAIResponseFlush_ReusedTypeKeepsSSEBytesAndTerminalSemantics(t *testing.T) { + tests := []struct { + name string + body string + flushCount int + }{ + { + name: "whitespace around done", + body: "data: \t[DONE] \n\n", + flushCount: 1, + }, + { + name: "invalid JSON before done", + body: "data: {\"type\":\n\ndata: [DONE]\n\n", + flushCount: 2, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + recorder := newOpenAIResponseFlushRecorder() + + result, err := runOpenAIResponseFlushTest(recorder, io.NopCloser(strings.NewReader(tt.body)), config.GatewayConfig{}) + + require.NoError(t, err) + require.NotNil(t, result) + gotBody, flushes := recorder.snapshot() + require.Equal(t, tt.body, gotBody) + require.Len(t, flushes, tt.flushCount) + }) + } +} + func TestOpenAIResponseFlush_ClientDisconnectStillDrainsUsage(t *testing.T) { first := "data: {\"type\":\"response.output_text.delta\",\"delta\":\"a\"}\n\n" terminal := "data: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":7,\"output_tokens\":5,\"input_tokens_details\":{\"cached_tokens\":2}}}}\n\n" diff --git a/backend/internal/service/openai_gateway_response_handling.go b/backend/internal/service/openai_gateway_response_handling.go index 8857bdd527..2acf0a9307 100644 --- a/backend/internal/service/openai_gateway_response_handling.go +++ b/backend/internal/service/openai_gateway_response_handling.go @@ -396,10 +396,12 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. // Extract data from SSE line (supports both "data: " and "data:" formats) if data, ok := extractOpenAISSEDataLine(line); ok { dataBytes := []byte(data) - if openAIStreamEventIsTerminal(data) { + eventTypeRaw := gjson.GetBytes(dataBytes, "type").String() + eventType := strings.TrimSpace(eventTypeRaw) + // 初始上游 data 的 type 只解析一次：原始值保持终止事件的精确匹配，规范化值供后续分支复用。 + if openAIStreamEventIsTerminalWithType(data, eventTypeRaw) { sawTerminalEvent = true } - eventType := strings.TrimSpace(gjson.GetBytes(dataBytes, "type").String()) if responseID == "" { responseID = extractOpenAIResponseIDFromJSONBytes(dataBytes) } diff --git a/backend/internal/service/openai_gateway_response_handling_type_test.go b/backend/internal/service/openai_gateway_response_handling_type_test.go new file mode 100644 index 0000000000..b783ab5c79 --- /dev/null +++ b/backend/internal/service/openai_gateway_response_handling_type_test.go @@ -0,0 +1,74 @@ +package service + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +func TestOpenAIStreamEventIsTerminalWithTypeMatchesExistingSemantics(t *testing.T) { + tests := []struct { + name string + data string + want bool + }{ + {name: "empty", data: "", want: false}, + {name: "whitespace", data: " \t ", want: false}, + {name: "done", data: " [DONE] ", want: true}, + {name: "JSON outer whitespace", data: " \n\t {\"type\":\"response.completed\"} \r\n", want: true}, + {name: "completed", data: `{"type":"response.completed"}`, want: true}, + {name: "response done", data: `{"type":"response.done"}`, want: true}, + {name: "failed", data: `{"type":"response.failed"}`, want: true}, + {name: "incomplete", data: `{"type":"response.incomplete"}`, want: true}, + {name: "cancelled", data: `{"type":"response.cancelled"}`, want: true}, + {name: "canceled", data: `{"type":"response.canceled"}`, want: true}, + {name: "delta", data: `{"type":"response.output_text.delta"}`, want: false}, + {name: "invalid JSON", data: `{"type":`, want: false}, + {name: "terminal with trailing garbage", data: `{"type":"response.completed"} trailing`, want: true}, + {name: "nonterminal with trailing garbage", data: `{"type":"response.output_text.delta"} trailing`, want: false}, + {name: "type whitespace remains nonterminal", data: `{"type":" response.completed "}`, want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + eventType := gjson.GetBytes([]byte(tt.data), "type").String() + got := openAIStreamEventIsTerminalWithType(tt.data, eventType) + + require.Equal(t, tt.want, got) + require.Equal(t, openAIStreamEventIsTerminal(tt.data), got) + }) + } +} + +var ( + benchmarkOpenAIResponseSSEEventTypeSink string + benchmarkOpenAIResponseSSETerminalSink bool +) + +func BenchmarkOpenAIResponseSSETypeExtraction(b *testing.B) { + data := `{"type":"response.output_text.delta","sequence_number":42,"delta":"streaming response benchmark payload"}` + dataBytes := []byte(data) + + b.Run("legacy double parse", func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(len(dataBytes))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + benchmarkOpenAIResponseSSETerminalSink = openAIStreamEventIsTerminal(data) + benchmarkOpenAIResponseSSEEventTypeSink = strings.TrimSpace(gjson.GetBytes(dataBytes, "type").String()) + } + }) + + b.Run("reused single parse", func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(len(dataBytes))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + eventTypeRaw := gjson.GetBytes(dataBytes, "type").String() + benchmarkOpenAIResponseSSEEventTypeSink = strings.TrimSpace(eventTypeRaw) + benchmarkOpenAIResponseSSETerminalSink = openAIStreamEventIsTerminalWithType(data, eventTypeRaw) + } + }) +} From c8ff478876ccdfc4be2041fee0456270f3a4c5e4 Mon Sep 17 00:00:00 2001 From: superman2003 <2112076433zcr@gmail.com> Date: Sun, 19 Jul 2026 18:59:50 +0800 Subject: [PATCH 23/59] fix(grok): default client tool caching on for Free accounts --- .../service/openai_gateway_grok_cache.go | 43 ++++++-- .../service/openai_gateway_grok_cache_test.go | 103 +++++++++++++++--- .../components/account/EditAccountModal.vue | 18 +-- .../EditAccountModal.grokUpstream.spec.ts | 46 +++++++- .../src/i18n/locales/en/admin/accounts.ts | 2 +- .../src/i18n/locales/zh/admin/accounts.ts | 2 +- 6 files changed, 176 insertions(+), 38 deletions(-) diff --git a/backend/internal/service/openai_gateway_grok_cache.go b/backend/internal/service/openai_gateway_grok_cache.go index ab81c16080..9fb916cacd 100644 --- a/backend/internal/service/openai_gateway_grok_cache.go +++ b/backend/internal/service/openai_gateway_grok_cache.go @@ -155,12 +155,12 @@ func hasGrokResponsesToolIntent(body []byte) bool { } // applyGrokFreeMessagesFunctionToolCacheRoute enables xAI's cache-capable -// mixed-tools route only for known Free accounts. Requests that already expose -// search capability can use the route automatically. Pure client-tool requests -// require an account-level opt-in because adding native search changes xAI's -// automatic tool selection behavior (#4486). +// mixed-tools route only for known Free accounts. Pure client tools default to +// the cache-capable route so an intermediate sub2api does not need to preserve +// client-specific opt-in headers. Operators can explicitly disable this per +// account when native search tools would change the desired behavior (#4486). func applyGrokFreeMessagesFunctionToolCacheRoute(body, intentSourceBody []byte, account *Account, cacheIdentity string) ([]byte, error) { - allowPureClientTools := account != nil && account.getExtraBool(grokClientToolCacheOptInExtraKey) + allowPureClientTools, _ := grokClientToolCacheAccountPolicy(account) return applyGrokFreeToolCacheRoute(body, intentSourceBody, account, cacheIdentity, allowPureClientTools, true) } @@ -168,7 +168,7 @@ func applyGrokFreeMessagesFunctionToolCacheRoute(body, intentSourceBody []byte, // sub2api header is consumed locally because buildGrokResponsesRequest only // forwards the explicitly supported OpenAI-Beta header from downstream. func applyGrokFreeRequestToolCacheRoute(c *gin.Context, body, intentSourceBody []byte, account *Account, cacheIdentity string) ([]byte, error) { - allowPureClientTools := account != nil && account.getExtraBool(grokClientToolCacheOptInExtraKey) + allowPureClientTools, accountPolicyExplicit := grokClientToolCacheAccountPolicy(account) requestOptOut := false if c != nil { switch strings.ToLower(strings.TrimSpace(c.GetHeader(grokClientToolCacheOptInHeader))) { @@ -179,16 +179,39 @@ func applyGrokFreeRequestToolCacheRoute(c *gin.Context, body, intentSourceBody [ requestOptOut = true } } - if !allowPureClientTools && !requestOptOut && isGrokClaudeDesktopResponsesCacheRequest(c) { + if !allowPureClientTools && !accountPolicyExplicit && !requestOptOut && isGrokClaudeDesktopResponsesCacheRequest(c) { allowPureClientTools = true } // A function merely named web_search/x_search is still a client function. - // Native Responses search types are safe to recognize automatically, while - // converting a client function requires either explicit opt-in or the strict - // Claude Desktop Responses fingerprint used above (#4486). + // Known Free OAuth accounts use the cache route by default; a request-scoped + // opt-in may override an account opt-out, while an explicit request opt-out + // always wins. The legacy Claude fingerprint remains only as a compatibility + // fallback when no account policy has been recorded (#4486). return applyGrokFreeToolCacheRoute(body, intentSourceBody, account, cacheIdentity, allowPureClientTools, allowPureClientTools) } +// grokClientToolCacheAccountPolicy is intentionally strict for configured +// values: only a JSON boolean is accepted. A missing key defaults on solely for +// accounts positively identified as Grok Free OAuth; paid, API-key, and unknown +// accounts remain fail-closed. +func grokClientToolCacheAccountPolicy(account *Account) (enabled, explicit bool) { + if !isKnownGrokFreeAccount(account) { + return false, false + } + if account.Extra == nil { + return true, false + } + value, exists := account.Extra[grokClientToolCacheOptInExtraKey] + if !exists { + return true, false + } + enabled, valid := value.(bool) + if !valid { + return false, true + } + return enabled, true +} + // isGrokClaudeDesktopResponsesCacheRequest recognizes the strict wire // fingerprint emitted when Claude Desktop's local agent is translated by // CC Switch into an OpenAI Responses request. Requiring every independent diff --git a/backend/internal/service/openai_gateway_grok_cache_test.go b/backend/internal/service/openai_gateway_grok_cache_test.go index d0d33ffb90..90155aadc6 100644 --- a/backend/internal/service/openai_gateway_grok_cache_test.go +++ b/backend/internal/service/openai_gateway_grok_cache_test.go @@ -197,7 +197,7 @@ func TestApplyGrokCacheIdentityWritesResponsesBodyAndHeader(t *testing.T) { require.False(t, gjson.GetBytes(unscopedBody, "tool_choice").Exists()) } -func TestApplyGrokCacheIdentityAppendsNativeToolsToResponseFunctions(t *testing.T) { +func TestGrokFreeMessagesClientToolCacheDefaultsOnForKnownFree(t *testing.T) { account := healthyGrokOAuthGatewayTestAccount(901, "access-token") account.Credentials["subscription_tier"] = " FREE " tests := []struct { @@ -211,7 +211,6 @@ func TestApplyGrokCacheIdentityAppendsNativeToolsToResponseFunctions(t *testing. for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // Pure client function tools without search → no native injection (#4486). intentBody := []byte(`{"model":"grok","tools":[{"type":"function","name":"lookup","description":"look up a value","parameters":{"type":"object"}},{"type":"function","name":"save","parameters":{"type":"object"}}]` + tt.toolChoiceJSON + `}`) body, err := applyGrokResponsesCacheIdentity(intentBody, intentBody, "isolated-id", true) require.NoError(t, err) @@ -220,16 +219,46 @@ func TestApplyGrokCacheIdentityAppendsNativeToolsToResponseFunctions(t *testing. require.NoError(t, err) require.Equal(t, "isolated-id", gjson.GetBytes(body, "prompt_cache_key").String()) tools := gjson.GetBytes(body, "tools").Array() - require.Len(t, tools, 2, "pure client functions should not get native search injected") + require.Len(t, tools, 4) require.Equal(t, "function", tools[0].Get("type").String()) require.Equal(t, "lookup", tools[0].Get("name").String()) require.Equal(t, "function", tools[1].Get("type").String()) require.Equal(t, "save", tools[1].Get("name").String()) + require.Equal(t, "web_search", tools[2].Get("type").String()) + require.Equal(t, "x_search", tools[3].Get("type").String()) require.Equal(t, tt.wantChoice, gjson.GetBytes(body, "tool_choice").Exists()) }) } } +func TestGrokFreeMessagesClientToolCacheDefaultsWithMissingAccountSetting(t *testing.T) { + body := []byte(`{"model":"grok","tools":[{"type":"function","name":"view_image","parameters":{"type":"object"}}],"tool_choice":"auto"}`) + tests := []struct { + name string + extra map[string]any + }{ + {name: "nil extra"}, + {name: "empty extra", extra: map[string]any{}}, + {name: "unrelated extra", extra: map[string]any{"other_setting": true}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + account := healthyGrokOAuthGatewayTestAccount(90101, "access-token") + account.Credentials["subscription_tier"] = "free" + account.Extra = tt.extra + + patched, err := applyGrokFreeMessagesFunctionToolCacheRoute(body, body, account, "isolated-id") + + require.NoError(t, err) + tools := gjson.GetBytes(patched, "tools").Array() + require.Len(t, tools, 3) + require.Equal(t, "web_search", tools[1].Get("type").String()) + require.Equal(t, "x_search", tools[2].Get("type").String()) + }) + } +} + func TestApplyGrokCacheIdentityAppendsNativeToolsWhenSearchPresent(t *testing.T) { account := healthyGrokOAuthGatewayTestAccount(901, "access-token") account.Credentials["subscription_tier"] = " FREE " @@ -268,9 +297,36 @@ func TestGrokFreeClientToolCacheAccountOptIn(t *testing.T) { require.Equal(t, "x_search", tools[3].Get("type").String()) } -func TestGrokFreeClientToolCacheRequestOptIn(t *testing.T) { +func TestGrokFreeMessagesClientToolCacheAccountOptOut(t *testing.T) { + body := []byte(`{"model":"grok","tools":[{"type":"function","name":"view_image","parameters":{"type":"object"}}],"tool_choice":"auto"}`) + tests := []struct { + name string + value any + }{ + {name: "explicit false", value: false}, + {name: "string false is malformed", value: "false"}, + {name: "numeric value is malformed", value: 1}, + {name: "null value is malformed", value: nil}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + account := healthyGrokOAuthGatewayTestAccount(90111, "access-token") + account.Credentials["subscription_tier"] = "free" + account.Extra = map[string]any{grokClientToolCacheOptInExtraKey: tt.value} + + patched, err := applyGrokFreeMessagesFunctionToolCacheRoute(body, body, account, "isolated-id") + + require.NoError(t, err) + require.JSONEq(t, string(body), string(patched)) + }) + } +} + +func TestGrokFreeClientToolCacheRequestOptInOverridesAccountOptOut(t *testing.T) { account := healthyGrokOAuthGatewayTestAccount(9014, "access-token") account.Credentials["subscription_tier"] = "free" + account.Extra = map[string]any{grokClientToolCacheOptInExtraKey: false} c := newGrokCacheTestContext(9014) c.Request.Header.Set(grokClientToolCacheOptInHeader, "prefer-cache") intentBody := []byte(`{"model":"grok","tools":[{"type":"function","name":"view_image","parameters":{"type":"object"}}],"tool_choice":"auto"}`) @@ -287,6 +343,23 @@ func TestGrokFreeClientToolCacheRequestOptIn(t *testing.T) { require.Equal(t, "x_search", tools[2].Get("type").String()) } +func TestGrokFreeChatRequestClientToolCacheDefaultsOnWithoutClientFingerprint(t *testing.T) { + account := healthyGrokOAuthGatewayTestAccount(90140, "access-token") + account.Credentials["subscription_tier"] = "free" + c := newGrokCacheTestContext(90140) + c.Request.URL.Path = "/v1/chat/completions" + body := []byte(`{"model":"grok","tools":[{"type":"function","name":"view_image","parameters":{"type":"object"}}],"tool_choice":"auto"}`) + + patched, err := applyGrokFreeRequestToolCacheRoute(c, body, body, account, "isolated-id") + + require.NoError(t, err) + tools := gjson.GetBytes(patched, "tools").Array() + require.Len(t, tools, 3) + require.Equal(t, "view_image", tools[0].Get("name").String()) + require.Equal(t, "web_search", tools[1].Get("type").String()) + require.Equal(t, "x_search", tools[2].Get("type").String()) +} + func TestGrokFreeClientToolCacheClaudeDesktopResponsesAutoOptIn(t *testing.T) { account := healthyGrokOAuthGatewayTestAccount(90141, "access-token") account.Credentials["subscription_tier"] = "free" @@ -318,6 +391,7 @@ func TestGrokFreeClientToolCacheClaudeDesktopResponsesAutoOptIn(t *testing.T) { func TestGrokFreeClientToolCacheClaudeDesktopFingerprintRequiresAllSignals(t *testing.T) { account := healthyGrokOAuthGatewayTestAccount(90142, "access-token") account.Credentials["subscription_tier"] = "free" + account.Extra = map[string]any{grokClientToolCacheOptInExtraKey: false} body := []byte(`{"model":"grok","tools":[{"type":"function","name":"Read","parameters":{"type":"object"}}],"tool_choice":"auto"}`) tests := []struct { @@ -416,19 +490,15 @@ func TestGrokFreeClientToolCacheClaudeDesktopFingerprintRequiresAllSignals(t *te } } -func TestGrokFreeClientToolCacheClaudeDesktopExplicitRequestOptOut(t *testing.T) { +func TestGrokFreeClientToolCacheExplicitRequestOptOut(t *testing.T) { account := healthyGrokOAuthGatewayTestAccount(90144, "access-token") account.Credentials["subscription_tier"] = "free" - account.Extra = map[string]any{grokClientToolCacheOptInExtraKey: true} body := []byte(`{"model":"grok","tools":[{"type":"function","name":"Read","parameters":{"type":"object"}}],"tool_choice":"auto"}`) for _, value := range []string{"0", "false", "no", "off"} { t.Run(value, func(t *testing.T) { c := newGrokCacheTestContext(90144) - c.Request.Header.Set("User-Agent", "claude-cli/2.1.215 (external, claude-desktop-3p, agent-sdk/0.3.215)") - c.Request.Header.Set("X-App", "cli") - c.Request.Header.Set("anthropic-client-platform", "desktop_app") - c.Request.Header.Set("X-Claude-Code-Session-Id", "desktop-session-1") + c.Request.URL.Path = "/v1/chat/completions" c.Request.Header.Set(grokClientToolCacheOptInHeader, value) patched, err := applyGrokFreeRequestToolCacheRoute(c, body, body, account, "isolated-id") @@ -455,7 +525,7 @@ func TestGrokFreeClientToolCacheClaudeDesktopAutoOptInDoesNotOverridePaidTier(t require.JSONEq(t, string(body), string(patched)) } -func TestGrokFreeRequestClientSearchFunctionRequiresOptIn(t *testing.T) { +func TestGrokFreeRequestClientSearchFunctionUsesDefaultAccountPolicy(t *testing.T) { account := healthyGrokOAuthGatewayTestAccount(9015, "access-token") account.Credentials["subscription_tier"] = "free" c := newGrokCacheTestContext(9015) @@ -464,7 +534,11 @@ func TestGrokFreeRequestClientSearchFunctionRequiresOptIn(t *testing.T) { patched, err := applyGrokFreeRequestToolCacheRoute(c, body, body, account, "isolated-id") require.NoError(t, err) - require.JSONEq(t, string(body), string(patched)) + tools := gjson.GetBytes(patched, "tools").Array() + require.Len(t, tools, 3) + require.Equal(t, "view_image", tools[0].Get("name").String()) + require.Equal(t, "web_search", tools[1].Get("type").String()) + require.Equal(t, "x_search", tools[2].Get("type").String()) } func TestGrokFreeRequestToolChoiceNoneUsesSafeCacheRoute(t *testing.T) { @@ -539,7 +613,7 @@ func TestApplyGrokCacheIdentityRequiresPatchedFunctionTools(t *testing.T) { {name: "missing tools", patchedBody: `{"model":"grok-4.5"}`}, {name: "empty tools", patchedBody: `{"model":"grok-4.5","tools":[]}`}, {name: "native tools only", patchedBody: `{"model":"grok-4.5","tools":[{"type":"web_search"}]}`}, - {name: "unexpected patched tool", patchedBody: `{"model":"grok-4.5","tools":[{"type":"function","name":"lookup"},{"type":"mcp","name":"server"}]}`}, + {name: "unexpected patched tool", patchedBody: `{"model":"grok-4.5","tools":[{"type":"function","name":"lookup"},{"type":"namespace","name":"server"}]}`}, } for _, tt := range tests { @@ -559,8 +633,7 @@ func TestApplyGrokCacheIdentityRequiresPatchedFunctionTools(t *testing.T) { } func TestGrokFreeMessagesFunctionToolCacheRouteRequiresKnownFreeTier(t *testing.T) { - // Include web_search as function to trigger native tool injection (pure client - // functions no longer trigger injection after #4486 fix). + // Include web_search as a function to also cover its conversion to a native tool. intentBody := []byte(`{"model":"grok","tools":[{"type":"function","name":"lookup"},{"type":"function","name":"web_search"}],"tool_choice":"auto"}`) tests := []struct { name string diff --git a/frontend/src/components/account/EditAccountModal.vue b/frontend/src/components/account/EditAccountModal.vue index 12afc93a11..df4838fd31 100644 --- a/frontend/src/components/account/EditAccountModal.vue +++ b/frontend/src/components/account/EditAccountModal.vue @@ -2768,7 +2768,9 @@ const headerOverrideCapable = computed( // Grok OAuth 自定义上游地址（仅转发端点；OAuth 授权/令牌刷新不受影响） const grokOAuthCustomBaseUrlEnabled = ref(false) const grokOAuthBaseUrl = ref('') -const grokClientToolCacheEnabled = ref(false) +// Grok Free OAuth accounts use client-tool prompt caching by default. Keep an +// explicit false in the account extra as the opt-out signal. +const grokClientToolCacheEnabled = ref(true) const interceptWarmupRequests = ref(false) const autoPauseOnExpired = ref(false) @@ -3424,10 +3426,14 @@ const syncFormFromAccount = (newAccount: Account | null) => { // Load Grok OAuth custom upstream URL state（存储的官方地址视同未定制） grokOAuthCustomBaseUrlEnabled.value = false grokOAuthBaseUrl.value = '' + const grokClientToolCacheSetting = + newAccount.platform === 'grok' && newAccount.type === 'oauth' + ? newAccount.extra?.[GROK_CLIENT_TOOL_CACHE_EXTRA_KEY] + : undefined grokClientToolCacheEnabled.value = newAccount.platform === 'grok' && newAccount.type === 'oauth' && - newAccount.extra?.[GROK_CLIENT_TOOL_CACHE_EXTRA_KEY] === true + (grokClientToolCacheSetting === undefined || grokClientToolCacheSetting === true) if (newAccount.platform === 'grok' && newAccount.type === 'oauth' && newAccount.credentials) { const grokCreds = newAccount.credentials as Record if (isCustomGrokBaseUrl(grokCreds.base_url)) { @@ -4318,11 +4324,9 @@ const handleSubmit = async () => { const newExtra: Record = { ...((props.account.extra as Record) || {}) } - if (grokClientToolCacheEnabled.value) { - newExtra[GROK_CLIENT_TOOL_CACHE_EXTRA_KEY] = true - } else { - delete newExtra[GROK_CLIENT_TOOL_CACHE_EXTRA_KEY] - } + // Persist both states so a disabled account remains opted out when the + // backend applies the default-enabled policy to missing values. + newExtra[GROK_CLIENT_TOOL_CACHE_EXTRA_KEY] = grokClientToolCacheEnabled.value updatePayload.extra = newExtra } diff --git a/frontend/src/components/account/__tests__/EditAccountModal.grokUpstream.spec.ts b/frontend/src/components/account/__tests__/EditAccountModal.grokUpstream.spec.ts index 068d7ac1a4..c96cd86d0c 100644 --- a/frontend/src/components/account/__tests__/EditAccountModal.grokUpstream.spec.ts +++ b/frontend/src/components/account/__tests__/EditAccountModal.grokUpstream.spec.ts @@ -264,18 +264,17 @@ describe('EditAccountModal Grok OAuth upstream config', () => { const payload = updateAccountMock.mock.calls[0]?.[1] expect(payload?.extra?.custom_setting).toBe('keep-me') - expect(payload?.extra).not.toHaveProperty('grok_client_tool_cache_enabled') + expect(payload?.extra?.grok_client_tool_cache_enabled).toBe(false) }) - it('enables client-tool caching while preserving unrelated extra fields', async () => { + it('defaults client-tool caching on when the setting is missing and persists explicit true', async () => { const account = buildGrokOAuthAccount({}, { custom_setting: 'keep-me' }) updateAccountMock.mockResolvedValue(account) const wrapper = mountModal(account) const toggle = wrapper.get('[data-testid="grok-client-tool-cache-toggle"]') - expect(toggle.attributes('aria-checked')).toBe('false') + expect(toggle.attributes('aria-checked')).toBe('true') - await toggle.trigger('click') await wrapper.get('form#edit-account-form').trigger('submit.prevent') await vi.waitFor(() => expect(updateAccountMock).toHaveBeenCalledTimes(1)) @@ -285,4 +284,43 @@ describe('EditAccountModal Grok OAuth upstream config', () => { custom_setting: 'keep-me' }) }) + + it('keeps an explicit false opt-out when saving an untouched account', async () => { + const account = buildGrokOAuthAccount( + {}, + { + grok_client_tool_cache_enabled: false, + custom_setting: 'keep-me' + } + ) + updateAccountMock.mockResolvedValue(account) + + const wrapper = mountModal(account) + const toggle = wrapper.get('[data-testid="grok-client-tool-cache-toggle"]') + expect(toggle.attributes('aria-checked')).toBe('false') + + await wrapper.get('form#edit-account-form').trigger('submit.prevent') + await vi.waitFor(() => expect(updateAccountMock).toHaveBeenCalledTimes(1)) + + const payload = updateAccountMock.mock.calls[0]?.[1] + expect(payload?.extra).toMatchObject({ + grok_client_tool_cache_enabled: false, + custom_setting: 'keep-me' + }) + }) + + it('shows malformed cache settings as disabled so the UI matches the fail-closed backend', () => { + const account = buildGrokOAuthAccount( + {}, + { + grok_client_tool_cache_enabled: 'true', + custom_setting: 'keep-me' + } + ) + + const wrapper = mountModal(account) + expect( + wrapper.get('[data-testid="grok-client-tool-cache-toggle"]').attributes('aria-checked') + ).toBe('false') + }) }) diff --git a/frontend/src/i18n/locales/en/admin/accounts.ts b/frontend/src/i18n/locales/en/admin/accounts.ts index 772161f6c8..9352f04c35 100644 --- a/frontend/src/i18n/locales/en/admin/accounts.ts +++ b/frontend/src/i18n/locales/en/admin/accounts.ts @@ -643,7 +643,7 @@ export default { }, grokClientToolCache: { title: 'Client Tool Cache (May Change Automatic Tool Selection)', - hint: 'For detected Grok Free OAuth accounts, enable upstream prompt caching for client function tools such as Codex and Trae. Enable only after confirming the automatic tool-selection behavior is acceptable.' + hint: 'For detected Grok Free OAuth accounts, this is enabled by default for client function tools such as Codex and Trae. Turn it off to opt out if the automatic tool-selection behavior is not acceptable.' }, autoPauseOnExpired: 'Auto Pause On Expired', autoPauseOnExpiredDesc: 'When enabled, the account will auto pause scheduling after it expires', diff --git a/frontend/src/i18n/locales/zh/admin/accounts.ts b/frontend/src/i18n/locales/zh/admin/accounts.ts index 9cfa1ee17c..d493b7aeff 100644 --- a/frontend/src/i18n/locales/zh/admin/accounts.ts +++ b/frontend/src/i18n/locales/zh/admin/accounts.ts @@ -736,7 +736,7 @@ export default { }, grokClientToolCache: { title: '客户端工具缓存（可能改变自动工具选择）', - hint: '仅对已识别为 Free 的 Grok OAuth 账号生效，为 Codex、Trae 等客户端函数工具请求启用上游提示缓存；请确认自动工具选择行为可接受后开启。' + hint: '仅对已识别为 Free 的 Grok OAuth 账号生效，默认会为 Codex、Trae 等客户端函数工具请求启用上游提示缓存；如不接受自动工具选择行为，可关闭此开关退出。' }, autoPauseOnExpired: '过期自动暂停调度', autoPauseOnExpiredDesc: '启用后，账号过期将自动暂停调度', From 3dcf412b6514a617fa7d709a1d3ddf1265a2e1f9 Mon Sep 17 00:00:00 2001 From: superman2003 <2112076433zcr@gmail.com> Date: Sun, 19 Jul 2026 19:00:08 +0800 Subject: [PATCH 24/59] fix(grok): preserve Trae cache routing across turns --- .../openai_gateway_grok_chat_bridge.go | 65 +++++++++++-- .../openai_gateway_grok_chat_bridge_test.go | 97 +++++++++++++++++++ 2 files changed, 156 insertions(+), 6 deletions(-) diff --git a/backend/internal/service/openai_gateway_grok_chat_bridge.go b/backend/internal/service/openai_gateway_grok_chat_bridge.go index baf1d8a592..89ebb4049e 100644 --- a/backend/internal/service/openai_gateway_grok_chat_bridge.go +++ b/backend/internal/service/openai_gateway_grok_chat_bridge.go @@ -22,18 +22,23 @@ const ( var grokChatResponsesBridgeTopLevelFields = map[string]struct{}{ "model": {}, "messages": {}, + "instructions": {}, "stream": {}, "stream_options": {}, "max_tokens": {}, "max_completion_tokens": {}, "temperature": {}, "top_p": {}, + "stop": {}, + "reasoning_effort": {}, "prompt_cache_key": {}, "tools": {}, "tool_choice": {}, "functions": {}, "function_call": {}, "parallel_tool_calls": {}, + "response_format": {}, + "service_tier": {}, } // grokChatResponsesBridgeEligibility deliberately accepts only request shapes @@ -46,11 +51,33 @@ func grokChatResponsesBridgeEligibility(body []byte) (bool, string) { return false, "invalid_json" } + // These fields have no effect when explicitly set to JSON null. Accepting + // that common SDK representation keeps the request on the bridge path, + // while non-null values remain unsupported because the Responses converter + // cannot preserve their Chat Completions semantics. for _, field := range []string{"stop", "reasoning_effort"} { - if _, exists := root[field]; exists { + if raw, exists := root[field]; exists && !grokChatJSONNull(raw) { return false, "unsupported_" + field } } + if raw, exists := root["instructions"]; exists { + var instructions string + if !grokChatJSONNull(raw) && json.Unmarshal(raw, &instructions) != nil { + return false, "invalid_instructions" + } + } + if raw, exists := root["response_format"]; exists { + var responseFormat map[string]json.RawMessage + if !grokChatJSONNull(raw) && (json.Unmarshal(raw, &responseFormat) != nil || responseFormat == nil) { + return false, "invalid_response_format" + } + } + if raw, exists := root["service_tier"]; exists { + var serviceTier string + if !grokChatJSONNull(raw) && json.Unmarshal(raw, &serviceTier) != nil { + return false, "invalid_service_tier" + } + } if raw, exists := root["tools"]; exists { if ok, reason := grokChatFunctionDeclarationsBridgeable(raw); !ok { return false, reason @@ -161,9 +188,16 @@ func grokChatResponsesBridgeEligibility(body []byte) (bool, string) { return false, reason } case "assistant": - if ok, reason := grokChatMessageFieldsBridgeable(message, "role", "content", "tool_calls"); !ok { + if ok, reason := grokChatMessageFieldsBridgeable(message, "role", "content", "reasoning_content", "tool_calls"); !ok { return false, reason } + reasoningContent := "" + if raw, exists := message["reasoning_content"]; exists { + if !grokChatJSONNull(raw) && json.Unmarshal(raw, &reasoningContent) != nil { + return false, "invalid_reasoning_content" + } + } + hasReasoningContent := strings.TrimSpace(reasoningContent) != "" toolCallCount := 0 if raw, exists := message["tool_calls"]; exists { var reason string @@ -174,20 +208,25 @@ func grokChatResponsesBridgeEligibility(body []byte) (bool, string) { } raw, hasContent := message["content"] if !hasContent || strings.TrimSpace(string(raw)) == "null" { - if toolCallCount == 0 { + if toolCallCount == 0 && !hasReasoningContent { return false, "non_text_message_content" } continue } var content string if json.Unmarshal(raw, &content) == nil { - if strings.TrimSpace(content) == "" && toolCallCount == 0 { + if strings.TrimSpace(content) == "" && toolCallCount == 0 && !hasReasoningContent { return false, "empty_message_content" } continue } if ok, reason := grokChatStructuredContentBridgeable(raw); !ok { - return false, reason + // The converter can still emit a standalone reasoning part when + // an otherwise empty content array accompanies reasoning_content. + // Do not broaden this exception to unsupported/malformed parts. + if !hasReasoningContent || reason != "empty_message_content" { + return false, reason + } } case "tool": if ok, reason := grokChatMessageFieldsBridgeable(message, "role", "content", "tool_call_id"); !ok { @@ -342,11 +381,17 @@ func grokChatAssistantToolCallsBridgeable(raw json.RawMessage) (int, string) { } for field := range call { switch field { - case "id", "type", "function": + case "id", "type", "function", "index": default: return 0, "unsafe_tool_call_field_" + field } } + if rawIndex, exists := call["index"]; exists { + var index *int + if json.Unmarshal(rawIndex, &index) != nil || (index != nil && *index < 0) { + return 0, "invalid_tool_call_index" + } + } var callID string if rawID, exists := call["id"]; !exists || json.Unmarshal(rawID, &callID) != nil || strings.TrimSpace(callID) == "" { return 0, "invalid_tool_call_id" @@ -419,6 +464,10 @@ func grokChatNullOrNone(raw json.RawMessage) bool { return json.Unmarshal(raw, &value) == nil && strings.EqualFold(strings.TrimSpace(value), "none") } +func grokChatJSONNull(raw json.RawMessage) bool { + return strings.TrimSpace(string(raw)) == "null" +} + func grokChatNullOrEmptyArray(raw json.RawMessage) bool { if strings.TrimSpace(string(raw)) == "null" { return true @@ -488,6 +537,10 @@ func (s *OpenAIGatewayService) forwardGrokChatCompletionsViaResponses( } responsesReq.Model = upstreamModel responsesReq.Stream = true + // Keep Chat and native Responses paths aligned for OpenAI-compatible + // service_tier aliases (for example, "fast" -> "priority"). Unknown + // values are omitted by the shared normalizer instead of reaching xAI. + normalizeResponsesRequestServiceTier(responsesReq) // These fields are useful to Codex but are not needed by the Grok CLI // protocol. Keep the bridge request as close as possible to native Grok. responsesReq.Include = nil diff --git a/backend/internal/service/openai_gateway_grok_chat_bridge_test.go b/backend/internal/service/openai_gateway_grok_chat_bridge_test.go index 6e39bae15d..188e38f702 100644 --- a/backend/internal/service/openai_gateway_grok_chat_bridge_test.go +++ b/backend/internal/service/openai_gateway_grok_chat_bridge_test.go @@ -39,6 +39,31 @@ func TestGrokChatResponsesBridgeEligibility(t *testing.T) { body: `{"model":"grok","messages":[{"role":"user","content":"hi"}],"stream":true,"stream_options":{"include_usage":true},"max_completion_tokens":256,"temperature":0.2,"top_p":0.9,"prompt_cache_key":"session","tools":[],"functions":null,"tool_choice":"none"}`, want: true, }, + { + name: "Trae compatible fields", + body: `{"model":"grok","messages":[{"role":"user","content":"return json"}],"instructions":"Be concise","response_format":{"type":"json_object"},"service_tier":"fast","stop":null,"reasoning_effort":null}`, + want: true, + }, + { + name: "Trae nullable SDK fields", + body: `{"model":"grok","messages":[{"role":"user","content":"hi"}],"instructions":null,"response_format":null,"service_tier":null,"stop":null,"reasoning_effort":null}`, + want: true, + }, + { + name: "invalid instructions fall back", + body: `{"model":"grok","messages":[{"role":"user","content":"hi"}],"instructions":{"text":"invalid"}}`, + reason: "invalid_instructions", + }, + { + name: "invalid response format falls back", + body: `{"model":"grok","messages":[{"role":"user","content":"hi"}],"response_format":"json_object"}`, + reason: "invalid_response_format", + }, + { + name: "invalid service tier falls back", + body: `{"model":"grok","messages":[{"role":"user","content":"hi"}],"service_tier":1}`, + reason: "invalid_service_tier", + }, { name: "stop falls back", body: `{"model":"grok","messages":[{"role":"user","content":"hi"}],"stop":"done"}`, @@ -99,6 +124,26 @@ func TestGrokChatResponsesBridgeEligibility(t *testing.T) { body: `{"model":"grok","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"call_lookup","type":"function","function":{"name":"lookup","arguments":"{\"key\":\"alpha\"}"}}]},{"role":"tool","tool_call_id":"call_lookup","content":"{\"value\":\"ok\"}"},{"role":"user","content":"summarize"}],"tools":[{"type":"function","function":{"name":"lookup","parameters":{"type":"object"}}}],"tool_choice":"auto","parallel_tool_calls":true}`, want: true, }, + { + name: "Trae reasoning and indexed tool history bridges", + body: `{"model":"grok","messages":[{"role":"assistant","content":null,"reasoning_content":"I should call lookup","tool_calls":[{"index":0,"id":"call_lookup","type":"function","function":{"name":"lookup","arguments":"{\"key\":\"alpha\"}"}}]},{"role":"tool","tool_call_id":"call_lookup","content":"{\"value\":\"ok\"}"},{"role":"user","content":"summarize"}],"tools":[{"type":"function","function":{"name":"lookup","parameters":{"type":"object"}}}]}`, + want: true, + }, + { + name: "reasoning only assistant history bridges", + body: `{"model":"grok","messages":[{"role":"assistant","reasoning_content":"Prior reasoning"},{"role":"user","content":"continue"}]}`, + want: true, + }, + { + name: "invalid reasoning content falls back", + body: `{"model":"grok","messages":[{"role":"assistant","reasoning_content":{"text":"invalid"}},{"role":"user","content":"continue"}]}`, + reason: "invalid_reasoning_content", + }, + { + name: "negative tool call index falls back", + body: `{"model":"grok","messages":[{"role":"assistant","content":null,"tool_calls":[{"index":-1,"id":"call_lookup","type":"function","function":{"name":"lookup","arguments":"{}"}}]}]}`, + reason: "invalid_tool_call_index", + }, { name: "unknown tool type falls back", body: `{"model":"grok","messages":[{"role":"user","content":"hi"}],"tools":[{"type":"web_search","function":{"name":"lookup","parameters":{"type":"object"}}}]}`, @@ -286,6 +331,58 @@ func TestForwardGrokChatViaResponsesTraeToolHistoryKeepsCacheRoute(t *testing.T) require.Equal(t, `{"value":"ok"}`, gjson.GetBytes(upstream.lastBody, "input.3.output").String()) } +func TestForwardGrokChatViaResponsesTraeCompatibilityFieldsKeepCacheRoute(t *testing.T) { + gin.SetMode(gin.TestMode) + + firstTurnBody := []byte(`{"model":"grok","messages":[{"role":"user","content":"Find alpha"}],"instructions":"Return concise JSON","stream":false,"response_format":{"type":"json_object"},"service_tier":"fast","stop":null,"reasoning_effort":null,"tools":[{"type":"function","function":{"name":"lookup","description":"Lookup a value","parameters":{"type":"object","properties":{"key":{"type":"string"}},"required":["key"]}}}],"tool_choice":"auto","parallel_tool_calls":true}`) + body := []byte(`{"model":"grok","messages":[{"role":"user","content":"Find alpha"},{"role":"assistant","content":null,"reasoning_content":"I should use lookup","tool_calls":[{"index":0,"id":"call_lookup","type":"function","function":{"name":"lookup","arguments":"{\"key\":\"alpha\"}"}}]},{"role":"tool","tool_call_id":"call_lookup","content":"{\"value\":\"ok\"}"},{"role":"user","content":"Summarize"}],"instructions":"Return concise JSON","stream":false,"response_format":{"type":"json_object"},"service_tier":"fast","stop":null,"reasoning_effort":null,"tools":[{"type":"function","function":{"name":"lookup","description":"Lookup a value","parameters":{"type":"object","properties":{"key":{"type":"string"}},"required":["key"]}}}],"tool_choice":"auto","parallel_tool_calls":true}`) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodPost, grokChatRawEndpoint, bytes.NewReader(body)) + c.Set("api_key", &APIKey{ID: 7161}) + + account := grokChatBridgeTestAccount(716) + account.Credentials["subscription_tier"] = "free" + repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{ + accountsByID: map[int64]*Account{account.ID: account}, + }} + upstream := &httpUpstreamRecorder{resp: grokChatBridgeCompletedResponse("resp_grok_chat_trae_compat", 12288)} + svc := &OpenAIGatewayService{ + httpUpstream: upstream, + grokTokenProvider: NewGrokTokenProvider(repo, nil), + accountRepo: repo, + } + + firstTurnIdentity := resolveGrokCacheIdentity(c, firstTurnBody, "", "grok-4.5") + extendedTurnIdentity := resolveGrokCacheIdentity(c, body, "", "grok-4.5") + require.NotEmpty(t, firstTurnIdentity) + require.Equal(t, firstTurnIdentity, extendedTurnIdentity) + + result, err := svc.ForwardAsChatCompletions(context.Background(), c, account, body, "", "") + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, xai.DefaultCLIBaseURL+"/responses", upstream.lastReq.URL.String()) + require.Equal(t, grokChatResponsesEndpoint, result.UpstreamEndpoint) + require.Equal(t, 12288, result.Usage.CacheReadInputTokens) + require.Equal(t, extendedTurnIdentity, gjson.GetBytes(upstream.lastBody, "prompt_cache_key").String()) + require.Equal(t, extendedTurnIdentity, upstream.lastReq.Header.Get(grokConversationIDHeader)) + require.Equal(t, "Return concise JSON", gjson.GetBytes(upstream.lastBody, "instructions").String()) + require.Equal(t, "json_object", gjson.GetBytes(upstream.lastBody, "text.format.type").String()) + require.Equal(t, "priority", gjson.GetBytes(upstream.lastBody, "service_tier").String()) + require.False(t, gjson.GetBytes(upstream.lastBody, "stop").Exists()) + require.False(t, gjson.GetBytes(upstream.lastBody, "reasoning").Exists()) + require.Contains(t, gjson.GetBytes(upstream.lastBody, "input.1.content.0.text").String(), "I should use lookup") + require.Equal(t, "function_call", gjson.GetBytes(upstream.lastBody, "input.2.type").String()) + require.Equal(t, "function_call_output", gjson.GetBytes(upstream.lastBody, "input.3.type").String()) + + tools := gjson.GetBytes(upstream.lastBody, "tools").Array() + require.Len(t, tools, 3) + require.Equal(t, "lookup", tools[0].Get("name").String()) + require.Equal(t, "web_search", tools[1].Get("type").String()) + require.Equal(t, "x_search", tools[2].Get("type").String()) + require.Equal(t, int64(12288), gjson.Get(recorder.Body.String(), "usage.prompt_tokens_details.cached_tokens").Int()) +} + func TestForwardGrokChatViaResponsesStreamingPropagatesCachedUsage(t *testing.T) { gin.SetMode(gin.TestMode) From 732aeef880103d36e37b1a4f0e843be22bcb4c3a Mon Sep 17 00:00:00 2001 From: Jlypx Date: Sun, 19 Jul 2026 19:41:01 +0800 Subject: [PATCH 25/59] =?UTF-8?q?fix:=20=E5=85=BC=E5=AE=B9=E5=8F=8D?= =?UTF-8?q?=E4=BB=A3=E5=92=8C=20Docker=20=E5=AE=A2=E6=88=B7=E7=AB=AF=20IP?= =?UTF-8?q?=20=E8=A7=A3=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- README_CN.md | 2 +- README_JA.md | 2 +- backend/internal/config/config.go | 20 +++++- backend/internal/config/config_test.go | 17 +++++ backend/internal/pkg/ip/ip.go | 87 ++++++++++++++++++++++++-- backend/internal/pkg/ip/ip_test.go | 20 ++++++ deploy/EDGE_SECURITY.md | 20 +++--- deploy/config.example.yaml | 15 ++++- 9 files changed, 165 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index f48e09087b..1e34840b00 100644 --- a/README.md +++ b/README.md @@ -571,7 +571,7 @@ Additional security-related options are available in `config.yaml`: - `security.response_headers.enabled` to enable configurable response header filtering (disabled uses default allowlist) - `security.csp` to control Content-Security-Policy headers - `billing.circuit_breaker` to fail closed on billing errors -- `server.trusted_proxies` to enable X-Forwarded-For parsing +- `server.trusted_proxies` to configure forwarded-IP trust for security-sensitive paths (local/container proxy ranges are trusted by default; replace them with exact remote proxy CIDRs when needed) - `turnstile.required` to require Turnstile in release mode **⚠️ Security Warning: HTTP URL Configuration** diff --git a/README_CN.md b/README_CN.md index bc33ad82a0..632b6532db 100644 --- a/README_CN.md +++ b/README_CN.md @@ -607,7 +607,7 @@ gateway: - `security.response_headers.enabled` 可启用可配置响应头过滤（关闭时使用默认白名单） - `security.csp` 配置 Content-Security-Policy - `billing.circuit_breaker` 计费异常时 fail-closed -- `server.trusted_proxies` 启用可信代理解析 X-Forwarded-For +- `server.trusted_proxies` 配置安全敏感路径的反代 IP 信任（本机/常见 Docker 私网网段默认已信任；远程反代请替换为精确 CIDR） - `turnstile.required` 在 release 模式强制启用 Turnstile **网关防御纵深建议（重点）** diff --git a/README_JA.md b/README_JA.md index 90541cadf4..849d951f68 100644 --- a/README_JA.md +++ b/README_JA.md @@ -569,7 +569,7 @@ default: - `security.response_headers.enabled` - 設定可能なレスポンスヘッダーフィルタリングを有効化（無効時はデフォルトの許可リストを使用） - `security.csp` - Content-Security-Policy ヘッダーの制御 - `billing.circuit_breaker` - 課金エラー時にフェイルクローズ -- `server.trusted_proxies` - X-Forwarded-For パースの有効化 +- `server.trusted_proxies` - セキュリティ用途の転送 IP 信頼を設定（ローカル/一般的な Docker プライベート範囲はデフォルトで信頼。リモートプロキシは正確な CIDR に置換） - `turnstile.required` - リリースモードでの Turnstile 必須化 **⚠️ セキュリティ警告: HTTP URL 設定** diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 38ebc94273..c7afd5594c 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -652,6 +652,21 @@ type ServerConfig struct { H2C H2CConfig `mapstructure:"h2c"` // HTTP/2 Cleartext 配置 } +// defaultTrustedProxies covers local and container-network reverse proxies. +// It keeps a fresh installation usable without weakening trust to every +// network address; deployments with a public/private load balancer should +// replace it with the exact proxy CIDRs. +func defaultTrustedProxies() []string { + return []string{ + "127.0.0.0/8", + "::1/128", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "fc00::/7", + } +} + // H2CConfig HTTP/2 Cleartext 配置 type H2CConfig struct { Enabled bool `mapstructure:"enabled"` // 是否启用 H2C @@ -1714,7 +1729,10 @@ func setDefaults() { viper.SetDefault("server.read_header_timeout", 10) // 10秒读取请求头 viper.SetDefault("server.max_header_bytes", 64*1024) viper.SetDefault("server.idle_timeout", 120) // 120秒空闲超时 - viper.SetDefault("server.trusted_proxies", []string{}) + // Trust local/container reverse proxies by default so existing deployments + // keep working without a config migration. An explicit list still replaces + // this default, and an explicit empty list disables the trust chain. + viper.SetDefault("server.trusted_proxies", defaultTrustedProxies()) viper.SetDefault("server.max_request_body_size", int64(256*1024*1024)) // H2C 默认配置 viper.SetDefault("server.h2c.enabled", false) diff --git a/backend/internal/config/config_test.go b/backend/internal/config/config_test.go index 36ae89ffa4..2356d724a1 100644 --- a/backend/internal/config/config_test.go +++ b/backend/internal/config/config_test.go @@ -41,12 +41,29 @@ func TestLoadHTTPIngressSafetyDefaults(t *testing.T) { require.NoError(t, err) require.Equal(t, 10, cfg.Server.ReadHeaderTimeout) require.Equal(t, 64*1024, cfg.Server.MaxHeaderBytes) + require.Equal(t, []string{ + "127.0.0.0/8", + "::1/128", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "fc00::/7", + }, cfg.Server.TrustedProxies) require.Equal(t, int64(32*1024*1024), cfg.Gateway.TextMaxBodySize) require.True(t, cfg.APIKeyAuth.InvalidAbuse.Enabled) require.Equal(t, 120, cfg.APIKeyAuth.InvalidAbuse.Threshold) require.Equal(t, 16384, cfg.APIKeyAuth.InvalidAbuse.Capacity) } +func TestLoadExplicitEmptyTrustedProxiesKeepsLegacyDefault(t *testing.T) { + resetViperWithJWTSecret(t) + viper.Set("server.trusted_proxies", []string{}) + + cfg, err := Load() + require.NoError(t, err) + require.Empty(t, cfg.Server.TrustedProxies) +} + func TestLoadForBootstrapAllowsMissingJWTSecret(t *testing.T) { viper.Reset() t.Setenv("JWT_SECRET", "") diff --git a/backend/internal/pkg/ip/ip.go b/backend/internal/pkg/ip/ip.go index b9c7774490..e24f11bff4 100644 --- a/backend/internal/pkg/ip/ip.go +++ b/backend/internal/pkg/ip/ip.go @@ -8,10 +8,51 @@ import ( "github.com/gin-gonic/gin" ) -// GetClientIP resolves a client address only through Gin's configured trusted -// proxy chain. Forwarding headers from a direct or untrusted peer are ignored. +// GetClientIP resolves the client address using the legacy forwarding-header +// precedence used before the trusted-proxy hardening. It remains the +// compatibility path for request metadata and usage/error logs; security- +// sensitive callers must use GetTrustedClientIP or GetSecurityClientIP. func GetClientIP(c *gin.Context) string { - return GetTrustedClientIP(c) + if c == nil { + return "" + } + + // Preserve the historical precedence used by existing reverse-proxy + // deployments, while skipping an internal proxy address when a public XFF + // value is available. This covers Docker/Nginx setups that accidentally + // write the bridge address into X-Real-IP. + var fallback string + if forwarded := normalizeIP(c.GetHeader("CF-Connecting-IP")); forwarded != "" { + fallback = forwarded + if !isPrivateIP(forwarded) { + return forwarded + } + } + if realIP := normalizeIP(c.GetHeader("X-Real-IP")); realIP != "" { + if fallback == "" { + fallback = realIP + } + if !isPrivateIP(realIP) { + return realIP + } + } + if xff := c.GetHeader("X-Forwarded-For"); xff != "" { + ips := strings.Split(xff, ",") + for _, candidate := range ips { + candidate = strings.TrimSpace(candidate) + if candidate != "" && !isPrivateIP(candidate) { + return normalizeIP(candidate) + } + } + if fallback == "" && len(ips) > 0 { + fallback = normalizeIP(strings.TrimSpace(ips[0])) + } + } + if fallback != "" { + return fallback + } + + return normalizeIP(c.ClientIP()) } // GetTrustedClientIP 从 Gin 的可信代理解析链提取客户端 IP。 @@ -24,9 +65,9 @@ func GetTrustedClientIP(c *gin.Context) string { return normalizeIP(c.ClientIP()) } -// GetSecurityClientIP returns the address resolved through Gin's configured -// trusted-proxy chain. The legacy toggle is retained for configuration/API -// compatibility, but never makes raw forwarding headers trustworthy by itself. +// GetSecurityClientIP returns the address used by security-sensitive paths. +// The legacy toggle remains in the signature for configuration/API +// compatibility, but cannot make raw forwarding headers trustworthy by itself. func GetSecurityClientIP(c *gin.Context, _ bool) string { return GetTrustedClientIP(c) } @@ -41,6 +82,27 @@ func normalizeIP(ip string) string { return ip } +// privateNets contains the private/loopback ranges skipped while selecting a +// public address from a legacy X-Forwarded-For chain. +var privateNets []*net.IPNet + +func init() { + for _, cidr := range []string{ + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "127.0.0.0/8", + "::1/128", + "fc00::/7", + } { + _, block, err := net.ParseCIDR(cidr) + if err != nil { + panic("invalid CIDR: " + cidr) + } + privateNets = append(privateNets, block) + } +} + // CompiledIPRules 表示预编译的 IP 匹配规则。 // PatternCount 记录原始规则数量，用于保留“规则存在但全无效”时的行为语义。 type CompiledIPRules struct { @@ -96,6 +158,19 @@ func matchesCompiledRules(parsedIP net.IP, rules *CompiledIPRules) bool { return false } +func isPrivateIP(ipStr string) bool { + ip := net.ParseIP(ipStr) + if ip == nil { + return false + } + for _, block := range privateNets { + if block.Contains(ip) { + return true + } + } + return false +} + // MatchesPattern 检查 IP 是否匹配指定的模式（支持单个 IP 或 CIDR）。 // pattern 可以是： // - 单个 IP: "192.168.1.100" diff --git a/backend/internal/pkg/ip/ip_test.go b/backend/internal/pkg/ip/ip_test.go index 2432abe7ab..f3ddddeb69 100644 --- a/backend/internal/pkg/ip/ip_test.go +++ b/backend/internal/pkg/ip/ip_test.go @@ -32,6 +32,26 @@ func TestGetTrustedClientIPUsesGinClientIP(t *testing.T) { require.Equal(t, "9.9.9.9", w.Body.String()) } +func TestGetClientIPPreservesLegacyDockerForwardedHeaders(t *testing.T) { + gin.SetMode(gin.TestMode) + + r := gin.New() + require.NoError(t, r.SetTrustedProxies(nil)) + r.GET("/t", func(c *gin.Context) { + c.String(200, GetClientIP(c)) + }) + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/t", nil) + req.RemoteAddr = "192.168.32.1:12345" + req.Header.Set("X-Forwarded-For", "10.0.0.2, 203.0.113.42") + req.Header.Set("X-Real-IP", "192.168.32.1") + r.ServeHTTP(w, req) + + require.Equal(t, 200, w.Code) + require.Equal(t, "203.0.113.42", w.Body.String()) +} + func TestCheckIPRestrictionWithCompiledRules(t *testing.T) { whitelist := CompileIPRules([]string{"10.0.0.0/8", "192.168.1.2"}) blacklist := CompileIPRules([]string{"10.1.1.1"}) diff --git a/deploy/EDGE_SECURITY.md b/deploy/EDGE_SECURITY.md index 0054253eb0..b2c08cf20c 100644 --- a/deploy/EDGE_SECURITY.md +++ b/deploy/EDGE_SECURITY.md @@ -29,15 +29,21 @@ the application's responsibility. ## Trusted client IPs -`server.trusted_proxies` must contain only the CIDR/IP addresses that connect -directly to Sub2API, normally the local Nginx/Caddy address or the private load -balancer subnet. An empty list disables forwarded-IP trust. +`server.trusted_proxies` controls forwarded-IP trust for security-sensitive +paths such as API-key ACLs, session binding, and rejection aggregation. Fresh +installations default to local/container ranges (`127.0.0.0/8`, `::1/128`, +`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, and `fc00::/7`) so a local +Nginx/Caddy or Docker bridge works without a migration. For a remote load +balancer, replace the defaults with only the CIDRs that connect directly to +Sub2API. An explicit empty list disables forwarded-IP trust for these paths; +ordinary request/usage metadata keeps its legacy compatibility behavior. -Never trust `CF-Connecting-IP`, `X-Real-IP`, or `X-Forwarded-For` merely because -the header exists. A CDN deployment must firewall the origin so only the CDN or -load balancer can reach it, and the proxy must overwrite forwarded headers. +Never use `CF-Connecting-IP`, `X-Real-IP`, or `X-Forwarded-For` for an ACL or +session decision merely because the header exists. A CDN deployment must +firewall the origin so only the CDN or load balancer can reach it, and the proxy +must overwrite forwarded headers. -Example for a proxy on the same host: +Example for a proxy on the same host (the default already covers this case): ```yaml server: diff --git a/deploy/config.example.yaml b/deploy/config.example.yaml index 24fa445158..2720c2d298 100644 --- a/deploy/config.example.yaml +++ b/deploy/config.example.yaml @@ -36,9 +36,18 @@ server: # Keep-alive idle timeout in seconds. # Keep-Alive 空闲连接超时（秒）。 idle_timeout: 120 - # Trusted proxies for X-Forwarded-For parsing (CIDR/IP). Empty disables trusted proxies. - # 信任的代理地址（CIDR/IP 格式），用于解析 X-Forwarded-For 头。留空则禁用代理信任。 - trusted_proxies: [] + # Trusted proxies for security-sensitive X-Forwarded-For parsing (CIDR/IP). + # These local/container ranges are the default; replace them with the exact + # proxy CIDRs for a remote load balancer. Set [] explicitly to disable trust. + # 安全敏感场景解析 X-Forwarded-For 的可信代理（CIDR/IP）。以下为本机/容器 + # 网段默认值；远程负载均衡请替换为实际 CIDR。显式设置 [] 可禁用代理信任。 + trusted_proxies: + - 127.0.0.0/8 + - ::1/128 + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + - fc00::/7 # Global max request body size in bytes (default: 256MB) # 全局最大请求体大小（字节，默认 256MB） # Applies to all requests, especially important for h2c first request memory protection From c044d5c1f1f244548525a9ea825c7e4221453153 Mon Sep 17 00:00:00 2001 From: orchief <33409040+orchief@users.noreply.github.com> Date: Sun, 19 Jul 2026 19:48:25 +0800 Subject: [PATCH 26/59] fix(openai): avoid duplicate OAuth system prompts Root cause: Chat bridge system messages were mirrored to instructions while remaining in input as developer messages. Fix: omit only losslessly promoted system messages for non-JSON-object OAuth Chat requests while preserving compatibility paths. Closes #4603 --- .../service/openai_codex_transform.go | 81 +++++++++++++++---- .../service/openai_codex_transform_test.go | 68 ++++++++++++++-- .../openai_gateway_chat_completions.go | 4 +- .../openai_gateway_chat_completions_test.go | 73 +++++++++++++++++ 4 files changed, 202 insertions(+), 24 deletions(-) diff --git a/backend/internal/service/openai_codex_transform.go b/backend/internal/service/openai_codex_transform.go index 6535454403..9d47c1c56e 100644 --- a/backend/internal/service/openai_codex_transform.go +++ b/backend/internal/service/openai_codex_transform.go @@ -77,10 +77,11 @@ type codexTransformResult struct { } type codexOAuthTransformOptions struct { - IsCodexCLI bool - IsCompact bool - SkipDefaultInstructions bool - PreserveToolCallIDs bool + IsCodexCLI bool + IsCompact bool + SkipDefaultInstructions bool + PreserveToolCallIDs bool + OmitPromotedSystemMessagesFromInput bool } const codexImageGenerationFunctionToolName = "image_gen.imagegen" @@ -219,9 +220,11 @@ func applyCodexOAuthTransformWithOptions(reqBody map[string]any, opts codexOAuth } // ChatGPT internal Codex endpoint does not accept role:"system". - // Keep the guidance in input as developer for Responses JSON mode, and - // also mirror it into instructions because Codex OAuth requires it. - if extractSystemMessagesFromInput(reqBody) { + // Mirror its text into instructions because Codex OAuth requires it. Some + // callers must also keep the guidance in input as developer (notably + // Responses JSON object mode), while Chat Completions compatibility can + // omit text-only messages after promoting them losslessly. + if extractSystemMessagesFromInput(reqBody, opts.OmitPromotedSystemMessagesFromInput) { result.Modified = true } @@ -1097,31 +1100,46 @@ func extractTextFromContent(content any) string { } } -// extractSystemMessagesFromInput scans input for role=="system", maps those -// items to developer, and mirrors their text into reqBody["instructions"]. -// It preserves the input items so Responses JSON mode can still see JSON -// instructions in input messages. -func extractSystemMessagesFromInput(reqBody map[string]any) bool { +// extractSystemMessagesFromInput scans input for role=="system" and mirrors +// their text into reqBody["instructions"]. By default it maps those items to +// developer so Responses JSON mode can still see JSON instructions in input. +// When omitPromoted is true, text-only items are removed after their content is +// losslessly promoted; mixed or malformed content is retained as developer. +func extractSystemMessagesFromInput(reqBody map[string]any, omitPromoted bool) bool { input, ok := reqBody["input"].([]any) if !ok || len(input) == 0 { return false } var systemTexts []string + filteredInput := make([]any, 0, len(input)) modified := false for _, item := range input { m, ok := item.(map[string]any) - if !ok { + if !ok || m["role"] != "system" { + filteredInput = append(filteredInput, item) continue } - if role, _ := m["role"].(string); role != "system" { - continue + + if omitPromoted { + if losslessText, lossless := extractLosslessTextFromContent(m["content"]); lossless { + if losslessText != "" { + systemTexts = append(systemTexts, losslessText) + } + modified = true + continue + } } - m["role"] = "developer" - modified = true + if text := extractTextFromContent(m["content"]); text != "" { systemTexts = append(systemTexts, text) } + m["role"] = "developer" + filteredInput = append(filteredInput, item) + modified = true + } + if omitPromoted && len(filteredInput) != len(input) { + reqBody["input"] = filteredInput } if len(systemTexts) == 0 { @@ -1137,6 +1155,35 @@ func extractSystemMessagesFromInput(reqBody map[string]any) bool { return true } +// extractLosslessTextFromContent returns text only when the entire content can +// be represented by an instructions string without dropping non-text parts. +func extractLosslessTextFromContent(content any) (string, bool) { + switch v := content.(type) { + case string: + return v, true + case []any: + var b strings.Builder + for _, part := range v { + m, ok := part.(map[string]any) + if !ok { + return "", false + } + typeName, ok := m["type"].(string) + if !ok || (typeName != "text" && typeName != "input_text" && typeName != "output_text") { + return "", false + } + text, ok := m["text"].(string) + if !ok { + return "", false + } + b.WriteString(text) + } + return b.String(), true + default: + return "", false + } +} + func extractPromptLikeInstructionsFromInput(reqBody map[string]any) string { input, ok := reqBody["input"].([]any) if !ok || len(input) == 0 { diff --git a/backend/internal/service/openai_codex_transform_test.go b/backend/internal/service/openai_codex_transform_test.go index 40d0fd4b41..14e3c0734b 100644 --- a/backend/internal/service/openai_codex_transform_test.go +++ b/backend/internal/service/openai_codex_transform_test.go @@ -1336,7 +1336,7 @@ func TestExtractSystemMessagesFromInput(t *testing.T) { map[string]any{"role": "user", "content": "hello"}, }, } - result := extractSystemMessagesFromInput(reqBody) + result := extractSystemMessagesFromInput(reqBody, false) require.False(t, result) input, ok := reqBody["input"].([]any) require.True(t, ok) @@ -1352,7 +1352,7 @@ func TestExtractSystemMessagesFromInput(t *testing.T) { map[string]any{"role": "user", "content": "hello"}, }, } - result := extractSystemMessagesFromInput(reqBody) + result := extractSystemMessagesFromInput(reqBody, false) require.True(t, result) input, ok := reqBody["input"].([]any) require.True(t, ok) @@ -1378,7 +1378,7 @@ func TestExtractSystemMessagesFromInput(t *testing.T) { }, }, } - result := extractSystemMessagesFromInput(reqBody) + result := extractSystemMessagesFromInput(reqBody, false) require.True(t, result) require.Equal(t, "Be helpful.", reqBody["instructions"]) input, ok := reqBody["input"].([]any) @@ -1401,7 +1401,7 @@ func TestExtractSystemMessagesFromInput(t *testing.T) { map[string]any{"role": "user", "content": "hi"}, }, } - result := extractSystemMessagesFromInput(reqBody) + result := extractSystemMessagesFromInput(reqBody, false) require.True(t, result) require.Equal(t, "First.\n\nSecond.", reqBody["instructions"]) input, ok := reqBody["input"].([]any) @@ -1427,7 +1427,7 @@ func TestExtractSystemMessagesFromInput(t *testing.T) { map[string]any{"role": "assistant", "content": "Hi there"}, }, } - result := extractSystemMessagesFromInput(reqBody) + result := extractSystemMessagesFromInput(reqBody, false) require.True(t, result) input, ok := reqBody["input"].([]any) require.True(t, ok) @@ -1452,7 +1452,7 @@ func TestExtractSystemMessagesFromInput(t *testing.T) { }, "instructions": "Existing instructions.", } - result := extractSystemMessagesFromInput(reqBody) + result := extractSystemMessagesFromInput(reqBody, false) require.True(t, result) require.Equal(t, "Extracted.\n\nExisting instructions.", reqBody["instructions"]) input, ok := reqBody["input"].([]any) @@ -1461,6 +1461,62 @@ func TestExtractSystemMessagesFromInput(t *testing.T) { require.True(t, ok) require.Equal(t, "developer", msg["role"]) }) + + t.Run("omit losslessly promoted text-only messages", func(t *testing.T) { + reqBody := map[string]any{ + "input": []any{ + map[string]any{"role": "system", "content": "First."}, + map[string]any{ + "role": "system", + "content": []any{ + map[string]any{"type": "text", "text": "Second "}, + map[string]any{"type": "input_text", "text": "and "}, + map[string]any{"type": "output_text", "text": "third."}, + }, + }, + map[string]any{"role": "user", "content": "hi"}, + }, + "instructions": "Existing.", + } + + result := extractSystemMessagesFromInput(reqBody, true) + + require.True(t, result) + require.Equal(t, "First.\n\nSecond and third.\n\nExisting.", reqBody["instructions"]) + input, ok := reqBody["input"].([]any) + require.True(t, ok) + require.Len(t, input, 1) + user, ok := input[0].(map[string]any) + require.True(t, ok) + require.Equal(t, "user", user["role"]) + }) + + t.Run("omit keeps mixed system content as developer", func(t *testing.T) { + reqBody := map[string]any{ + "input": []any{ + map[string]any{ + "role": "system", + "content": []any{ + map[string]any{"type": "input_text", "text": "Inspect this image."}, + map[string]any{"type": "input_image", "image_url": "https://example.com/image.png"}, + }, + }, + map[string]any{"role": "user", "content": "hi"}, + }, + } + + result := extractSystemMessagesFromInput(reqBody, true) + + require.True(t, result) + require.Equal(t, "Inspect this image.", reqBody["instructions"]) + input, ok := reqBody["input"].([]any) + require.True(t, ok) + require.Len(t, input, 2) + developer, ok := input[0].(map[string]any) + require.True(t, ok) + require.Equal(t, "developer", developer["role"]) + require.Len(t, developer["content"], 2) + }) } // TestApplyCodexOAuthTransform_StripsPromptCacheRetention is a regression diff --git a/backend/internal/service/openai_gateway_chat_completions.go b/backend/internal/service/openai_gateway_chat_completions.go index cccec46056..8a8349b5f8 100644 --- a/backend/internal/service/openai_gateway_chat_completions.go +++ b/backend/internal/service/openai_gateway_chat_completions.go @@ -196,8 +196,10 @@ func (s *OpenAIGatewayService) ForwardAsChatCompletions( if err := json.Unmarshal(responsesBody, &reqBody); err != nil { return nil, fmt.Errorf("unmarshal for codex transform: %w", err) } + isJSONObjectFormat := strings.EqualFold(strings.TrimSpace(gjson.GetBytes(responsesBody, "text.format.type").String()), "json_object") codexResult := applyCodexOAuthTransformWithOptions(reqBody, codexOAuthTransformOptions{ - SkipDefaultInstructions: !isResponsesShape, + SkipDefaultInstructions: !isResponsesShape, + OmitPromotedSystemMessagesFromInput: !isResponsesShape && !isJSONObjectFormat, }) if !isResponsesShape { ensureCodexOAuthInstructionsField(reqBody) diff --git a/backend/internal/service/openai_gateway_chat_completions_test.go b/backend/internal/service/openai_gateway_chat_completions_test.go index b307aa4ede..ededa9f322 100644 --- a/backend/internal/service/openai_gateway_chat_completions_test.go +++ b/backend/internal/service/openai_gateway_chat_completions_test.go @@ -279,6 +279,79 @@ func TestForwardAsChatCompletions_OAuthDoesNotInjectDefaultInstructions(t *testi require.NotContains(t, string(upstream.lastBody), "Communicate with the user by streaming thinking") } +func forwardOAuthChatCompletionsForUpstreamBody(t *testing.T, body []byte) []byte { + t.Helper() + + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(body)) + c.Request.Header.Set("Content-Type", "application/json") + + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusBadRequest, + Header: http.Header{"Content-Type": []string{"application/json"}, "x-request-id": []string{"rid_chat_system_promotion"}}, + Body: io.NopCloser(strings.NewReader(`{"error":{"type":"invalid_request_error","message":"stop before response parsing"}}`)), + }} + svc := &OpenAIGatewayService{cfg: &config.Config{}, httpUpstream: upstream} + account := &Account{ + ID: 4, + Name: "openai-oauth", + Platform: PlatformOpenAI, + Type: AccountTypeOAuth, + Concurrency: 1, + Credentials: map[string]any{ + "access_token": "oauth-token", + "chatgpt_account_id": "chatgpt-acc", + }, + } + + result, err := svc.ForwardAsChatCompletions(context.Background(), c, account, body, "", "gpt-5.4") + require.Error(t, err) + require.Nil(t, result) + require.NotEmpty(t, upstream.lastBody) + return upstream.lastBody +} + +func TestForwardAsChatCompletions_OAuthPromotesSystemMessageWithoutDuplication(t *testing.T) { + const systemPrompt = "Unique system prefix for token accounting." + body := []byte(`{"model":"gpt-5.4","messages":[{"role":"system","content":"` + systemPrompt + `"},{"role":"user","content":"hello"}],"stream":false}`) + + upstreamBody := forwardOAuthChatCompletionsForUpstreamBody(t, body) + + require.Equal(t, systemPrompt, gjson.GetBytes(upstreamBody, "instructions").String()) + require.Equal(t, int64(1), gjson.GetBytes(upstreamBody, "input.#").Int()) + require.Equal(t, "user", gjson.GetBytes(upstreamBody, "input.0.role").String()) + require.Equal(t, 1, strings.Count(string(upstreamBody), systemPrompt)) +} + +func TestForwardAsChatCompletions_OAuthJsonObjectKeepsSystemMessageInInput(t *testing.T) { + const systemPrompt = "Return JSON only." + body := []byte(`{"model":"gpt-5.4","messages":[{"role":"system","content":"` + systemPrompt + `"},{"role":"user","content":"symbol data"}],"response_format":{"type":" JSON_OBJECT "},"stream":false}`) + + upstreamBody := forwardOAuthChatCompletionsForUpstreamBody(t, body) + + require.Equal(t, systemPrompt, gjson.GetBytes(upstreamBody, "instructions").String()) + require.Equal(t, int64(2), gjson.GetBytes(upstreamBody, "input.#").Int()) + require.Equal(t, "developer", gjson.GetBytes(upstreamBody, "input.0.role").String()) + require.Equal(t, systemPrompt, gjson.GetBytes(upstreamBody, "input.0.content").String()) + require.Equal(t, 2, strings.Count(string(upstreamBody), systemPrompt)) +} + +func TestForwardAsChatCompletions_OAuthKeepsMixedSystemContentInInput(t *testing.T) { + const systemPrompt = "Inspect this reference image." + body := []byte(`{"model":"gpt-5.4","messages":[{"role":"system","content":[{"type":"text","text":"` + systemPrompt + `"},{"type":"image_url","image_url":{"url":"https://example.com/reference.png"}}]},{"role":"user","content":"hello"}],"stream":false}`) + + upstreamBody := forwardOAuthChatCompletionsForUpstreamBody(t, body) + + require.Equal(t, systemPrompt, gjson.GetBytes(upstreamBody, "instructions").String()) + require.Equal(t, int64(2), gjson.GetBytes(upstreamBody, "input.#").Int()) + require.Equal(t, "developer", gjson.GetBytes(upstreamBody, "input.0.role").String()) + require.Equal(t, int64(2), gjson.GetBytes(upstreamBody, "input.0.content.#").Int()) + require.Equal(t, "input_image", gjson.GetBytes(upstreamBody, "input.0.content.1.type").String()) + require.Equal(t, "https://example.com/reference.png", gjson.GetBytes(upstreamBody, "input.0.content.1.image_url").String()) +} + func TestForwardAsChatCompletions_ClientDisconnectDrainsUpstreamUsage(t *testing.T) { gin.SetMode(gin.TestMode) From 3ed2873f14d0430a3cf2abf082f1a861586bf254 Mon Sep 17 00:00:00 2001 From: superman2003 <2112076433zcr@gmail.com> Date: Sun, 19 Jul 2026 21:13:27 +0800 Subject: [PATCH 27/59] fix(openai): mark OAuth accounts unschedulable on Codex models manifest 401 The Codex models manifest path returned upstream 401s straight to the client without feeding them into the account state machinery. A revoked or invalidated OAuth account therefore stayed active and schedulable, kept being selected for subsequent /models requests, and produced repeated 502s until an admin ran a manual connection test (#4544). - Route ChatGPT-backend manifest 401s through the shared upstream-error handling: token cache invalidation, temp-unschedulable cooldown for refreshable OAuth accounts, permanent disable for token_revoked/token_invalidated, plus the runtime scheduling block. - Treat ChatGPT-backend manifest 401s as failover-eligible so the current /models request can switch to a healthy account instead of returning 502. Custom API key upstream 401s keep the existing no-failover, no-disable behavior since their /models auth is not authoritative for the account. - Skip Agent Identity accounts: their 401s can be task-scoped and have a dedicated recovery flow. - Attach the selected account to the ops error-log context so /models failures record the account_id. Fixes #4544 Co-authored-by: Cursor --- .../handler/openai_codex_models_handler.go | 2 + .../service/openai_codex_models_service.go | 45 +++++- .../openai_codex_models_service_test.go | 141 ++++++++++++++++++ 3 files changed, 186 insertions(+), 2 deletions(-) diff --git a/backend/internal/handler/openai_codex_models_handler.go b/backend/internal/handler/openai_codex_models_handler.go index 1c1357cbfa..ba96ac3923 100644 --- a/backend/internal/handler/openai_codex_models_handler.go +++ b/backend/internal/handler/openai_codex_models_handler.go @@ -53,6 +53,8 @@ func (h *OpenAIGatewayHandler) CodexModels(c *gin.Context) { h.errorResponse(c, http.StatusServiceUnavailable, "upstream_error", "No available OpenAI accounts") return } + // 让 ops 错误日志携带实际选中的上游账号，便于定位失效账号（#4544）。 + setOpsSelectedAccount(c, account.ID, account.Platform) manifest, err := h.gatewayService.FetchCodexModelsManifest(c.Request.Context(), account, c.Query("client_version"), c.GetHeader("If-None-Match")) if err != nil { diff --git a/backend/internal/service/openai_codex_models_service.go b/backend/internal/service/openai_codex_models_service.go index 32a9bd57d3..c43b0d8376 100644 --- a/backend/internal/service/openai_codex_models_service.go +++ b/backend/internal/service/openai_codex_models_service.go @@ -47,6 +47,7 @@ type codexModelsManifestUpstreamError struct { err error retryable bool statusCode int + headers http.Header body []byte } @@ -56,7 +57,13 @@ func (e *codexModelsManifestUpstreamError) Unwrap() error { return e.err } // IsRetryableCodexModelsManifestError reports whether another selected account // may succeed without changing the request. Configuration and upstream 4xx -// responses, except 429, are intentionally not retried. +// responses, except 429 and ChatGPT-backend 401, are intentionally not +// retried. A manifest 401 from the ChatGPT Codex backend reflects the selected +// OAuth account's upstream token rather than the client request (the client's +// own API key was already validated locally), so a different account may still +// serve the manifest. Custom API key upstreams keep the old no-failover 401 +// behavior because their /models auth semantics are not authoritative for the +// account. func IsRetryableCodexModelsManifestError(err error) bool { var upstreamErr *codexModelsManifestUpstreamError return errors.As(err, &upstreamErr) && upstreamErr.retryable @@ -322,6 +329,7 @@ func (s *OpenAIGatewayService) FetchCodexModelsManifest(ctx context.Context, acc } manifest, fetchErr := s.fetchCodexModelsManifestUpstream(ctx, request, ifNoneMatch) if !credAccount.IsOpenAIAgentIdentity() || !isAgentIdentityTaskInvalidCodexModelsError(fetchErr) { + s.handleCodexModelsManifestAccountAuthError(ctx, account, credAccount, fetchErr) return manifest, fetchErr } expectedTaskID := strings.TrimSpace(credAccount.GetCredential("task_id")) @@ -349,6 +357,37 @@ func isAgentIdentityTaskInvalidCodexModelsError(err error) bool { isAgentIdentityTaskInvalidHTTPResponse(upstreamErr.statusCode, upstreamErr.body) } +// handleCodexModelsManifestAccountAuthError feeds manifest 401s from the +// ChatGPT Codex backend into the shared upstream-error state machinery +// (token cache invalidation, temp-unschedulable cooldown, or permanent +// disable for token_revoked/token_invalidated). Without this, an account +// whose OAuth token was revoked upstream stays active and schedulable and +// keeps being selected for every subsequent /models request (#4544). +// +// Scope is deliberately limited to plain OAuth accounts: the manifest +// endpoint authenticates with the same token as /responses forwarding, so a +// 401 is authoritative for the account. Agent Identity accounts are excluded +// because their 401s can be task-scoped and have a dedicated recovery flow, +// and API key manifests come from custom upstreams whose /models auth may +// diverge from their chat endpoints. +func (s *OpenAIGatewayService) handleCodexModelsManifestAccountAuthError(ctx context.Context, account, credAccount *Account, err error) { + if s == nil || account == nil || err == nil { + return + } + if credAccount == nil || !credAccount.IsOpenAIOAuth() || credAccount.IsOpenAIAgentIdentity() { + return + } + var upstreamErr *codexModelsManifestUpstreamError + if !errors.As(err, &upstreamErr) || upstreamErr.statusCode != http.StatusUnauthorized { + return + } + headers := upstreamErr.headers + if headers == nil { + headers = http.Header{} + } + s.handleOpenAIAccountUpstreamError(ctx, account, upstreamErr.statusCode, headers, upstreamErr.body) +} + func (s *OpenAIGatewayService) fetchCachedAPIKeyCodexModelsManifest(ctx context.Context, request codexModelsManifestRequest, ifNoneMatch string) (*CodexModelsManifest, error) { if err := ctx.Err(); err != nil { return nil, err @@ -450,8 +489,10 @@ func (s *OpenAIGatewayService) fetchCodexModelsManifestUpstream(ctx context.Cont return nil, &codexModelsManifestUpstreamError{ err: infraerrors.Newf(http.StatusBadGateway, "OPENAI_CODEX_MODELS_UPSTREAM_FAILED", "codex models manifest upstream error %d: %s", resp.StatusCode, message), statusCode: resp.StatusCode, + headers: resp.Header.Clone(), body: body, - retryable: resp.StatusCode == http.StatusTooManyRequests || + retryable: (resp.StatusCode == http.StatusUnauthorized && !request.useAPIKeyUpstream) || + resp.StatusCode == http.StatusTooManyRequests || (resp.StatusCode >= http.StatusInternalServerError && resp.StatusCode < 600), } } diff --git a/backend/internal/service/openai_codex_models_service_test.go b/backend/internal/service/openai_codex_models_service_test.go index 3075993680..2be2519d2f 100644 --- a/backend/internal/service/openai_codex_models_service_test.go +++ b/backend/internal/service/openai_codex_models_service_test.go @@ -1094,6 +1094,147 @@ func TestFetchCodexModelsManifestAPIKeyRejectsBaseURLFragment(t *testing.T) { } } +// codexModelsAccountStateRepo records account state transitions triggered by +// manifest upstream errors (#4544). +type codexModelsAccountStateRepo struct { + AccountRepository + mu sync.Mutex + setErrorCalls int + lastErrorMsg string + setTempUnschedCalls int + lastTempReason string +} + +func (r *codexModelsAccountStateRepo) SetError(_ context.Context, _ int64, errorMsg string) error { + r.mu.Lock() + defer r.mu.Unlock() + r.setErrorCalls++ + r.lastErrorMsg = errorMsg + return nil +} + +func (r *codexModelsAccountStateRepo) SetTempUnschedulable(_ context.Context, _ int64, _ time.Time, reason string) error { + r.mu.Lock() + defer r.mu.Unlock() + r.setTempUnschedCalls++ + r.lastTempReason = reason + return nil +} + +func newCodexModels401TestService(repo AccountRepository) *OpenAIGatewayService { + rateLimitService := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) + s := &OpenAIGatewayService{rateLimitService: rateLimitService} + rateLimitService.SetAccountRuntimeBlocker(s) + return s +} + +func TestFetchCodexModelsManifestOAuth401MarksAccountUnschedulable(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte(`{"detail":{"message":"invalid token"}}`)) + })) + defer server.Close() + + original := chatgptCodexModelsURL + chatgptCodexModelsURL = server.URL + defer func() { chatgptCodexModelsURL = original }() + + repo := &codexModelsAccountStateRepo{} + s := newCodexModels401TestService(repo) + account := newCodexModelsTestAccount() + account.Credentials["refresh_token"] = "test-refresh-token" + + _, err := s.FetchCodexModelsManifest(context.Background(), account, "0.137.0", "") + require.Error(t, err) + require.True(t, IsRetryableCodexModelsManifestError(err), "manifest 401 should allow account failover") + require.Equal(t, 1, repo.setTempUnschedCalls, "OAuth 401 should temp-unschedule the account") + require.Equal(t, 0, repo.setErrorCalls) + require.True(t, s.isOpenAIAccountRuntimeBlocked(account), "account should be runtime-blocked after manifest 401") +} + +func TestFetchCodexModelsManifestOAuth401TokenRevokedDisablesAccount(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte(`{"error":{"code":"token_revoked","message":"token has been revoked"}}`)) + })) + defer server.Close() + + original := chatgptCodexModelsURL + chatgptCodexModelsURL = server.URL + defer func() { chatgptCodexModelsURL = original }() + + repo := &codexModelsAccountStateRepo{} + s := newCodexModels401TestService(repo) + account := newCodexModelsTestAccount() + account.Credentials["refresh_token"] = "test-refresh-token" + + _, err := s.FetchCodexModelsManifest(context.Background(), account, "0.137.0", "") + require.Error(t, err) + require.True(t, IsRetryableCodexModelsManifestError(err)) + require.Equal(t, 1, repo.setErrorCalls, "revoked token should permanently disable the account") + require.Contains(t, repo.lastErrorMsg, "Token revoked") + require.Equal(t, 0, repo.setTempUnschedCalls) +} + +func TestFetchCodexModelsManifestAgentIdentity401DoesNotDisableAccount(t *testing.T) { + key, privateKey := newTestAgentIdentityKey(t) + account := &Account{ + ID: 6, + Platform: PlatformOpenAI, + Type: AccountTypeOAuth, + Credentials: map[string]any{ + "auth_mode": OpenAIAuthModeAgentIdentity, + "agent_runtime_id": key.runtimeID, + "agent_private_key": privateKey, + "task_id": key.taskID, + "chatgpt_account_id": "acc-agent-401", + }, + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte(`{"detail":"some non-task 401"}`)) + })) + defer server.Close() + + original := chatgptCodexModelsURL + chatgptCodexModelsURL = server.URL + defer func() { chatgptCodexModelsURL = original }() + + repo := &codexModelsAccountStateRepo{} + s := newCodexModels401TestService(repo) + + _, err := s.FetchCodexModelsManifest(context.Background(), account, "0.137.0", "") + require.Error(t, err) + require.Equal(t, 0, repo.setErrorCalls, "agent identity 401s must not disable the account") + require.Equal(t, 0, repo.setTempUnschedCalls) +} + +func TestFetchCodexModelsManifestAPIKey401KeepsNoFailoverAndNoDisable(t *testing.T) { + upstream := &codexModelsHTTPUpstreamStub{do: func(_ *http.Request, _ string, _ int64, _ int) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusUnauthorized, + Status: "401 Unauthorized", + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"error":"invalid api key"}`)), + }, nil + }} + + repo := &codexModelsAccountStateRepo{} + s := newCodexModelsAPIKeyTestService(upstream) + s.rateLimitService = NewRateLimitService(repo, nil, &config.Config{}, nil, nil) + + _, err := s.FetchCodexModelsManifest( + context.Background(), + newCodexModelsAPIKeyTestAccount("https://upstream.example"), + "0.144.0", + "", + ) + require.Error(t, err) + require.False(t, IsRetryableCodexModelsManifestError(err), "custom upstream manifest 401 keeps the no-failover behavior") + require.Equal(t, 0, repo.setErrorCalls, "custom upstream manifest 401 must not disable the account") + require.Equal(t, 0, repo.setTempUnschedCalls) +} + func TestFetchCodexModelsManifestAPIKeyUpstreamError(t *testing.T) { upstream := &codexModelsHTTPUpstreamStub{do: func(_ *http.Request, _ string, _ int64, _ int) (*http.Response, error) { return &http.Response{ From 3a4966f786bfced43d0866c561dd760e9ab44019 Mon Sep 17 00:00:00 2001 From: cat Date: Sun, 19 Jul 2026 21:28:47 +0800 Subject: [PATCH 28/59] =?UTF-8?q?fix(openai):=20=E9=99=90=E5=88=B6=20Codex?= =?UTF-8?q?=20call=5Fid=20=E5=BD=92=E4=B8=80=E5=8C=96=E9=95=BF=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/openai_codex_transform.go | 38 +++++++++-- .../service/openai_codex_transform_test.go | 64 +++++++++++++++++++ 2 files changed, 95 insertions(+), 7 deletions(-) diff --git a/backend/internal/service/openai_codex_transform.go b/backend/internal/service/openai_codex_transform.go index 6535454403..11e34439cd 100644 --- a/backend/internal/service/openai_codex_transform.go +++ b/backend/internal/service/openai_codex_transform.go @@ -1,6 +1,8 @@ package service import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "strings" @@ -83,6 +85,34 @@ type codexOAuthTransformOptions struct { PreserveToolCallIDs bool } +const ( + codexCallIDMaxLength = 64 + codexCallIDPrefix = "fc_" +) + +func normalizeCodexCallID(id string) string { + candidate := id + switch { + case id == "": + return "" + case strings.HasPrefix(id, "fc"): + case strings.HasPrefix(id, "call_"): + candidate = codexCallIDPrefix + strings.TrimPrefix(id, "call_") + default: + candidate = codexCallIDPrefix + id + } + if len(candidate) <= codexCallIDMaxLength { + return candidate + } + return compactCodexCallID(candidate) +} + +func compactCodexCallID(id string) string { + digest := sha256.Sum256([]byte("sub2api:codex-call-id:v1:" + id)) + encoded := hex.EncodeToString(digest[:]) + return codexCallIDPrefix + encoded[:codexCallIDMaxLength-len(codexCallIDPrefix)] +} + const codexImageGenerationFunctionToolName = "image_gen.imagegen" const ( @@ -1334,13 +1364,7 @@ func filterCodexInputWithOptions(input []any, opts codexInputFilterOptions) []an if opts.PreserveCallIDs { return id } - if id == "" || strings.HasPrefix(id, "fc") { - return id - } - if strings.HasPrefix(id, "call_") { - return "fc_" + strings.TrimPrefix(id, "call_") - } - return "fc_" + id + return normalizeCodexCallID(id) } if typ == "item_reference" { diff --git a/backend/internal/service/openai_codex_transform_test.go b/backend/internal/service/openai_codex_transform_test.go index 40d0fd4b41..c184f49d7d 100644 --- a/backend/internal/service/openai_codex_transform_test.go +++ b/backend/internal/service/openai_codex_transform_test.go @@ -127,6 +127,70 @@ func TestApplyCodexOAuthTransform_ToolContinuationNormalizesToolReferenceIDsOnly require.Equal(t, "fc_1", second["call_id"]) } +func TestApplyCodexOAuthTransform_BoundsLongCallIDsAndPreservesPairing(t *testing.T) { + suffix := strings.Repeat("z", 62) + for _, tc := range []struct { + name string + callID string + outputCallID string + }{ + {name: "non-native boundary id", callID: "call-" + strings.Repeat("x", 59), outputCallID: "call-" + strings.Repeat("x", 59)}, + {name: "overlong fc id", callID: "fc_" + strings.Repeat("y", 62), outputCallID: "fc_" + strings.Repeat("y", 62)}, + {name: "equivalent prefixes", callID: "call_" + suffix, outputCallID: "fc_" + suffix}, + } { + t.Run(tc.name, func(t *testing.T) { + reqBody := map[string]any{ + "model": "gpt-5.2", + "input": []any{ + map[string]any{"type": "function_call", "call_id": tc.callID, "name": "shell"}, + map[string]any{"type": "function_call_output", "call_id": tc.outputCallID, "output": "done"}, + }, + } + + applyCodexOAuthTransform(reqBody, false, false) + + input, ok := reqBody["input"].([]any) + require.True(t, ok) + call, ok := input[0].(map[string]any) + require.True(t, ok) + output, ok := input[1].(map[string]any) + require.True(t, ok) + + fixedCallID, ok := call["call_id"].(string) + require.True(t, ok) + require.LessOrEqual(t, len(fixedCallID), codexCallIDMaxLength) + require.True(t, strings.HasPrefix(fixedCallID, codexCallIDPrefix)) + require.Equal(t, fixedCallID, output["call_id"]) + require.Equal(t, fixedCallID, normalizeCodexCallID(tc.callID)) + require.Equal(t, fixedCallID, normalizeCodexCallID(tc.outputCallID)) + }) + } +} + +func TestApplyCodexOAuthTransform_PreservesLongCallIDsWhenRequested(t *testing.T) { + callID := "call-" + strings.Repeat("x", 70) + reqBody := map[string]any{ + "model": "gpt-5.2", + "input": []any{ + map[string]any{"type": "function_call", "call_id": callID, "name": "shell"}, + map[string]any{"type": "function_call_output", "call_id": callID, "output": "done"}, + }, + } + + applyCodexOAuthTransformWithOptions(reqBody, codexOAuthTransformOptions{ + PreserveToolCallIDs: true, + }) + + input, ok := reqBody["input"].([]any) + require.True(t, ok) + call, ok := input[0].(map[string]any) + require.True(t, ok) + output, ok := input[1].(map[string]any) + require.True(t, ok) + require.Equal(t, callID, call["call_id"]) + require.Equal(t, callID, output["call_id"]) +} + func TestApplyCodexOAuthTransform_ToolSearchOutputPreservesCallID(t *testing.T) { reqBody := map[string]any{ "model": "gpt-5.2", From d0fa8c63f171459e9878bd8a88a30fcc999035f5 Mon Sep 17 00:00:00 2001 From: cugxuan Date: Sun, 19 Jul 2026 21:37:26 +0800 Subject: [PATCH 29/59] fix(subscription): round remaining days up --- .../subscription_calculate_progress_test.go | 2 +- backend/internal/service/user_subscription.go | 16 +++++++- .../user_subscription_days_remaining_test.go | 37 +++++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 backend/internal/service/user_subscription_days_remaining_test.go diff --git a/backend/internal/service/subscription_calculate_progress_test.go b/backend/internal/service/subscription_calculate_progress_test.go index 650522d54c..8a3f8bee8e 100644 --- a/backend/internal/service/subscription_calculate_progress_test.go +++ b/backend/internal/service/subscription_calculate_progress_test.go @@ -34,7 +34,7 @@ func TestCalculateProgress_BasicFields(t *testing.T) { assert.Equal(t, int64(100), progress.ID) assert.Equal(t, "Premium", progress.GroupName) assert.Equal(t, sub.ExpiresAt, progress.ExpiresAt) - assert.True(t, progress.ExpiresInDays == 29 || progress.ExpiresInDays == 30, "ExpiresInDays should be 29 or 30, got %d", progress.ExpiresInDays) + assert.Equal(t, 30, progress.ExpiresInDays) assert.Nil(t, progress.Daily, "无日限额时 Daily 应为 nil") assert.Nil(t, progress.Weekly, "无周限额时 Weekly 应为 nil") assert.Nil(t, progress.Monthly, "无月限额时 Monthly 应为 nil") diff --git a/backend/internal/service/user_subscription.go b/backend/internal/service/user_subscription.go index 732401ada2..12e3dd428a 100644 --- a/backend/internal/service/user_subscription.go +++ b/backend/internal/service/user_subscription.go @@ -2,6 +2,8 @@ package service import "time" +const subscriptionDayDuration = 24 * time.Hour + type UserSubscription struct { ID int64 UserID int64 @@ -41,10 +43,20 @@ func (s *UserSubscription) IsExpired() bool { } func (s *UserSubscription) DaysRemaining() int { - if s.IsExpired() { + return s.daysRemainingAt(time.Now()) +} + +func (s *UserSubscription) daysRemainingAt(now time.Time) int { + remaining := s.ExpiresAt.Sub(now) + if remaining <= 0 { return 0 } - return int(time.Until(s.ExpiresAt).Hours() / 24) + + days := int(remaining / subscriptionDayDuration) + if remaining%subscriptionDayDuration != 0 { + days++ + } + return days } func (s *UserSubscription) IsWindowActivated() bool { diff --git a/backend/internal/service/user_subscription_days_remaining_test.go b/backend/internal/service/user_subscription_days_remaining_test.go new file mode 100644 index 0000000000..571d250f73 --- /dev/null +++ b/backend/internal/service/user_subscription_days_remaining_test.go @@ -0,0 +1,37 @@ +//go:build unit + +package service + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestUserSubscriptionDaysRemainingAt(t *testing.T) { + now := time.Date(2026, time.July, 19, 12, 0, 0, 0, time.UTC) + tests := []struct { + name string + expiresAt time.Time + want int + }{ + {name: "expired", expiresAt: now.Add(-time.Nanosecond), want: 0}, + {name: "expires now", expiresAt: now, want: 0}, + {name: "less than one day", expiresAt: now.Add(subscriptionDayDuration - time.Nanosecond), want: 1}, + {name: "exactly one day", expiresAt: now.Add(subscriptionDayDuration), want: 1}, + {name: "over one day", expiresAt: now.Add(subscriptionDayDuration + time.Nanosecond), want: 2}, + {name: "less than two days", expiresAt: now.Add(2*subscriptionDayDuration - time.Nanosecond), want: 2}, + {name: "exactly two days", expiresAt: now.Add(2 * subscriptionDayDuration), want: 2}, + {name: "over two days", expiresAt: now.Add(2*subscriptionDayDuration + time.Nanosecond), want: 3}, + {name: "exactly seven days", expiresAt: now.Add(7 * subscriptionDayDuration), want: 7}, + {name: "over seven days", expiresAt: now.Add(7*subscriptionDayDuration + time.Nanosecond), want: 8}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sub := &UserSubscription{ExpiresAt: tt.expiresAt} + require.Equal(t, tt.want, sub.daysRemainingAt(now)) + }) + } +} From 8a147fcc513f9bc83cd8b964a16fcb90da48e05e Mon Sep 17 00:00:00 2001 From: Jlypx Date: Sun, 19 Jul 2026 21:39:24 +0800 Subject: [PATCH 30/59] =?UTF-8?q?fix:=20=E8=A7=A3=E6=9E=90=E6=98=BE?= =?UTF-8?q?=E5=BC=8F=E5=8F=AF=E4=BF=A1=E4=BB=A3=E7=90=86=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- backend/internal/config/config.go | 73 ++++++++++++------------- backend/internal/config/config_test.go | 76 +++++++++++++++++++++++--- 2 files changed, 102 insertions(+), 47 deletions(-) diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index c7afd5594c..6aef080721 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -639,32 +639,18 @@ type PricingConfig struct { } type ServerConfig struct { - Host string `mapstructure:"host"` - Port int `mapstructure:"port"` - Mode string `mapstructure:"mode"` // debug/release - EnableServerTiming bool `mapstructure:"enable_server_timing"` // Admin UI Server-Timing response header - FrontendURL string `mapstructure:"frontend_url"` // 前端基础 URL，用于生成邮件中的外部链接 - ReadHeaderTimeout int `mapstructure:"read_header_timeout"` // 读取请求头超时（秒） - MaxHeaderBytes int `mapstructure:"max_header_bytes"` // 请求头最大字节数（HTTP/2 映射为 header-list 上限） - IdleTimeout int `mapstructure:"idle_timeout"` // 空闲连接超时（秒） - TrustedProxies []string `mapstructure:"trusted_proxies"` // 可信代理列表（CIDR/IP） - MaxRequestBodySize int64 `mapstructure:"max_request_body_size"` // 全局最大请求体限制 - H2C H2CConfig `mapstructure:"h2c"` // HTTP/2 Cleartext 配置 -} - -// defaultTrustedProxies covers local and container-network reverse proxies. -// It keeps a fresh installation usable without weakening trust to every -// network address; deployments with a public/private load balancer should -// replace it with the exact proxy CIDRs. -func defaultTrustedProxies() []string { - return []string{ - "127.0.0.0/8", - "::1/128", - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16", - "fc00::/7", - } + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + Mode string `mapstructure:"mode"` // debug/release + EnableServerTiming bool `mapstructure:"enable_server_timing"` // Admin UI Server-Timing response header + FrontendURL string `mapstructure:"frontend_url"` // 前端基础 URL，用于生成邮件中的外部链接 + ReadHeaderTimeout int `mapstructure:"read_header_timeout"` // 读取请求头超时（秒） + MaxHeaderBytes int `mapstructure:"max_header_bytes"` // 请求头最大字节数（HTTP/2 映射为 header-list 上限） + IdleTimeout int `mapstructure:"idle_timeout"` // 空闲连接超时（秒） + TrustedProxies []string `mapstructure:"trusted_proxies"` // 可信代理列表（CIDR/IP） + TrustedProxiesConfigured bool `mapstructure:"-" json:"-" yaml:"-"` // 是否显式配置了可信代理列表 + MaxRequestBodySize int64 `mapstructure:"max_request_body_size"` // 全局最大请求体限制 + H2C H2CConfig `mapstructure:"h2c"` // HTTP/2 Cleartext 配置 } // H2CConfig HTTP/2 Cleartext 配置 @@ -683,13 +669,15 @@ type CORSConfig struct { } type SecurityConfig struct { - URLAllowlist URLAllowlistConfig `mapstructure:"url_allowlist"` - ResponseHeaders ResponseHeaderConfig `mapstructure:"response_headers"` - CSP CSPConfig `mapstructure:"csp"` - ProxyFallback ProxyFallbackConfig `mapstructure:"proxy_fallback"` - ProxyProbe ProxyProbeConfig `mapstructure:"proxy_probe"` - TrustForwardedIPForAPIKeyACL bool `mapstructure:"trust_forwarded_ip_for_api_key_acl"` - trustForwardedIPForAPIKeyACLLive *atomic.Bool `mapstructure:"-"` + URLAllowlist URLAllowlistConfig `mapstructure:"url_allowlist"` + ResponseHeaders ResponseHeaderConfig `mapstructure:"response_headers"` + CSP CSPConfig `mapstructure:"csp"` + ProxyFallback ProxyFallbackConfig `mapstructure:"proxy_fallback"` + ProxyProbe ProxyProbeConfig `mapstructure:"proxy_probe"` + // TrustForwardedIPForAPIKeyACL enables legacy raw forwarded-header takeover. + // When disabled, server.trusted_proxies is authoritative for all client-IP consumers. + TrustForwardedIPForAPIKeyACL bool `mapstructure:"trust_forwarded_ip_for_api_key_acl"` + trustForwardedIPForAPIKeyACLLive *atomic.Bool `mapstructure:"-"` } func (c *Config) TrustForwardedIPForAPIKeyACL() bool { @@ -703,6 +691,12 @@ func (c *Config) TrustForwardedIPForAPIKeyACL() bool { return live.Load() } +// ForwardedClientIPTrustEnabled reports whether the legacy forwarded-header +// compatibility mode currently overrides server.trusted_proxies. +func (c *Config) ForwardedClientIPTrustEnabled() bool { + return c != nil && c.TrustForwardedIPForAPIKeyACL() +} + func (c *Config) SetTrustForwardedIPForAPIKeyACL(enabled bool) { if c == nil { return @@ -1579,11 +1573,18 @@ func load(allowMissingJWTSecret bool) (*Config, error) { } // 配置文件不存在时使用默认值 } + trustedProxiesEnv, trustedProxiesEnvConfigured := os.LookupEnv("SERVER_TRUSTED_PROXIES") + trustedProxiesConfigured := viper.InConfig("server.trusted_proxies") || + viper.IsSet("server.trusted_proxies") || trustedProxiesEnvConfigured var cfg Config if err := viper.Unmarshal(&cfg); err != nil { return nil, fmt.Errorf("unmarshal config error: %w", err) } + if trustedProxiesEnvConfigured { + cfg.Server.TrustedProxies = normalizeStringSlice(strings.Split(trustedProxiesEnv, ",")) + } + cfg.Server.TrustedProxiesConfigured = trustedProxiesConfigured if cfg.Gateway.OpenAIScheduler.StickyEscapeTTFTMs == 0 { cfg.Gateway.OpenAIScheduler.StickyEscapeTTFTMs = 15000 } @@ -1729,10 +1730,6 @@ func setDefaults() { viper.SetDefault("server.read_header_timeout", 10) // 10秒读取请求头 viper.SetDefault("server.max_header_bytes", 64*1024) viper.SetDefault("server.idle_timeout", 120) // 120秒空闲超时 - // Trust local/container reverse proxies by default so existing deployments - // keep working without a config migration. An explicit list still replaces - // this default, and an explicit empty list disables the trust chain. - viper.SetDefault("server.trusted_proxies", defaultTrustedProxies()) viper.SetDefault("server.max_request_body_size", int64(256*1024*1024)) // H2C 默认配置 viper.SetDefault("server.h2c.enabled", false) @@ -1789,7 +1786,7 @@ func setDefaults() { viper.SetDefault("security.csp.enabled", true) viper.SetDefault("security.csp.policy", DefaultCSPPolicy) viper.SetDefault("security.proxy_probe.insecure_skip_verify", false) - viper.SetDefault("security.trust_forwarded_ip_for_api_key_acl", false) + viper.SetDefault("security.trust_forwarded_ip_for_api_key_acl", true) // Security - disable direct fallback on proxy error viper.SetDefault("security.proxy_fallback.allow_direct_on_error", false) diff --git a/backend/internal/config/config_test.go b/backend/internal/config/config_test.go index 2356d724a1..20da10a6a7 100644 --- a/backend/internal/config/config_test.go +++ b/backend/internal/config/config_test.go @@ -41,27 +41,85 @@ func TestLoadHTTPIngressSafetyDefaults(t *testing.T) { require.NoError(t, err) require.Equal(t, 10, cfg.Server.ReadHeaderTimeout) require.Equal(t, 64*1024, cfg.Server.MaxHeaderBytes) - require.Equal(t, []string{ - "127.0.0.0/8", - "::1/128", - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16", - "fc00::/7", - }, cfg.Server.TrustedProxies) + require.Empty(t, cfg.Server.TrustedProxies) + require.False(t, cfg.Server.TrustedProxiesConfigured) + require.True(t, cfg.TrustForwardedIPForAPIKeyACL()) require.Equal(t, int64(32*1024*1024), cfg.Gateway.TextMaxBodySize) require.True(t, cfg.APIKeyAuth.InvalidAbuse.Enabled) require.Equal(t, 120, cfg.APIKeyAuth.InvalidAbuse.Threshold) require.Equal(t, 16384, cfg.APIKeyAuth.InvalidAbuse.Capacity) } -func TestLoadExplicitEmptyTrustedProxiesKeepsLegacyDefault(t *testing.T) { +func TestLoadExplicitEmptyTrustedProxiesEnablesConfiguredMode(t *testing.T) { resetViperWithJWTSecret(t) viper.Set("server.trusted_proxies", []string{}) cfg, err := Load() require.NoError(t, err) require.Empty(t, cfg.Server.TrustedProxies) + require.True(t, cfg.Server.TrustedProxiesConfigured) +} + +func TestLoadExplicitTrustedProxiesEnablesConfiguredMode(t *testing.T) { + resetViperWithJWTSecret(t) + viper.Set("server.trusted_proxies", []string{"127.0.0.1/32"}) + + cfg, err := Load() + require.NoError(t, err) + require.Equal(t, []string{"127.0.0.1/32"}, cfg.Server.TrustedProxies) + require.True(t, cfg.Server.TrustedProxiesConfigured) +} + +func TestLoadTrustedProxiesFromEnvironment(t *testing.T) { + resetViperWithJWTSecret(t) + t.Setenv("SERVER_TRUSTED_PROXIES", "127.0.0.1/32, ::1/128") + + cfg, err := Load() + require.NoError(t, err) + require.Equal(t, []string{"127.0.0.1/32", "::1/128"}, cfg.Server.TrustedProxies) + require.True(t, cfg.Server.TrustedProxiesConfigured) +} + +func TestLoadExplicitEmptyTrustedProxiesFromEnvironment(t *testing.T) { + resetViperWithJWTSecret(t) + t.Setenv("SERVER_TRUSTED_PROXIES", "") + + cfg, err := Load() + require.NoError(t, err) + require.Empty(t, cfg.Server.TrustedProxies) + require.True(t, cfg.Server.TrustedProxiesConfigured) +} + +func TestLoadTrustedProxiesPresenceFromYAML(t *testing.T) { + tests := []struct { + name string + yaml string + want []string + configured bool + }{ + {name: "absent", yaml: "server:\n mode: debug\n", configured: false}, + {name: "explicit empty", yaml: "server:\n trusted_proxies: []\n", want: []string{}, configured: true}, + { + name: "populated", + yaml: "server:\n trusted_proxies:\n - 127.0.0.1/32\n - ::1/128\n", + want: []string{"127.0.0.1/32", "::1/128"}, + configured: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + resetViperWithJWTSecret(t) + configDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(configDir, "config.yaml"), []byte(test.yaml), 0o600)) + t.Setenv("DATA_DIR", configDir) + + cfg, err := Load() + require.NoError(t, err) + require.Equal(t, test.want, cfg.Server.TrustedProxies) + require.Equal(t, test.configured, cfg.Server.TrustedProxiesConfigured) + }) + } } func TestLoadForBootstrapAllowsMissingJWTSecret(t *testing.T) { From 6e2ba0e4cdbfde86c6d85f67638adb6a8b87f0fe Mon Sep 17 00:00:00 2001 From: Jlypx Date: Sun, 19 Jul 2026 21:39:49 +0800 Subject: [PATCH 31/59] =?UTF-8?q?fix:=20=E7=BB=9F=E4=B8=80=E8=AF=B7?= =?UTF-8?q?=E6=B1=82=E7=BA=A7=E5=AE=A2=E6=88=B7=E7=AB=AF=20IP=20=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- backend/internal/pkg/ip/ip.go | 36 +++++++++++- backend/internal/pkg/ip/ip_test.go | 92 +++++++++++++++++++++--------- 2 files changed, 99 insertions(+), 29 deletions(-) diff --git a/backend/internal/pkg/ip/ip.go b/backend/internal/pkg/ip/ip.go index e24f11bff4..2e74665b0b 100644 --- a/backend/internal/pkg/ip/ip.go +++ b/backend/internal/pkg/ip/ip.go @@ -8,6 +8,24 @@ import ( "github.com/gin-gonic/gin" ) +const legacyForwardedIPTrustKey = "sub2api.legacy_forwarded_ip_trust" + +// SetLegacyForwardedIPTrust records whether raw forwarding headers override +// Gin's server.trusted_proxies chain for this request. +func SetLegacyForwardedIPTrust(c *gin.Context, enabled bool) { + if c != nil { + c.Set(legacyForwardedIPTrustKey, enabled) + } +} + +func requestUsesLegacyForwardedIPTrust(c *gin.Context) bool { + if c == nil { + return true + } + enabled, ok := c.Get(legacyForwardedIPTrustKey) + return !ok || enabled == true +} + // GetClientIP resolves the client address using the legacy forwarding-header // precedence used before the trusted-proxy hardening. It remains the // compatibility path for request metadata and usage/error logs; security- @@ -16,6 +34,9 @@ func GetClientIP(c *gin.Context) string { if c == nil { return "" } + if !requestUsesLegacyForwardedIPTrust(c) { + return GetTrustedClientIP(c) + } // Preserve the historical precedence used by existing reverse-proxy // deployments, while skipping an internal proxy address when a public XFF @@ -66,9 +87,18 @@ func GetTrustedClientIP(c *gin.Context) string { } // GetSecurityClientIP returns the address used by security-sensitive paths. -// The legacy toggle remains in the signature for configuration/API -// compatibility, but cannot make raw forwarding headers trustworthy by itself. -func GetSecurityClientIP(c *gin.Context, _ bool) string { +// When legacy forwarded-IP trust is enabled, raw forwarding headers take over +// client-IP resolution. When disabled, Gin's server.trusted_proxies chain is +// authoritative. +func GetSecurityClientIP(c *gin.Context, trustForwarded bool) string { + if c != nil { + if requestTrust, ok := c.Get(legacyForwardedIPTrustKey); ok { + trustForwarded = requestTrust == true + } + } + if trustForwarded { + return GetClientIP(c) + } return GetTrustedClientIP(c) } diff --git a/backend/internal/pkg/ip/ip_test.go b/backend/internal/pkg/ip/ip_test.go index f3ddddeb69..d4347cab5a 100644 --- a/backend/internal/pkg/ip/ip_test.go +++ b/backend/internal/pkg/ip/ip_test.go @@ -73,41 +73,30 @@ func TestCheckIPRestrictionWithCompiledRules_InvalidWhitelistStillDenies(t *test require.Equal(t, "access denied", reason) } -func TestGetSecurityClientIPNeverTrustsHeadersFromUntrustedPeer(t *testing.T) { +func TestGetSecurityClientIPSwitchEnabledUsesLegacyHeaders(t *testing.T) { gin.SetMode(gin.TestMode) - for _, tc := range []struct { - name string - trustForwarded bool - want string - }{ - {name: "legacy toggle disabled", trustForwarded: false, want: "9.9.9.9"}, - {name: "legacy toggle enabled", trustForwarded: true, want: "9.9.9.9"}, - } { - t.Run(tc.name, func(t *testing.T) { - r := gin.New() - require.NoError(t, r.SetTrustedProxies(nil)) - r.GET("/t", func(c *gin.Context) { - c.String(200, GetSecurityClientIP(c, tc.trustForwarded)) - }) + r := gin.New() + require.NoError(t, r.SetTrustedProxies(nil)) + r.GET("/t", func(c *gin.Context) { + c.String(200, GetSecurityClientIP(c, true)) + }) - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/t", nil) - req.RemoteAddr = "9.9.9.9:12345" - req.Header.Set("X-Real-IP", "1.2.3.4") - r.ServeHTTP(w, req) + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/t", nil) + req.RemoteAddr = "9.9.9.9:12345" + req.Header.Set("X-Real-IP", "1.2.3.4") + r.ServeHTTP(w, req) - require.Equal(t, 200, w.Code) - require.Equal(t, tc.want, w.Body.String()) - }) - } + require.Equal(t, 200, w.Code) + require.Equal(t, "1.2.3.4", w.Body.String()) } -func TestGetSecurityClientIPUsesConfiguredTrustedProxy(t *testing.T) { +func TestGetSecurityClientIPSwitchDisabledUsesConfiguredTrustedProxy(t *testing.T) { gin.SetMode(gin.TestMode) r := gin.New() require.NoError(t, r.SetTrustedProxies([]string{"9.9.9.9"})) - r.GET("/t", func(c *gin.Context) { c.String(200, GetSecurityClientIP(c, true)) }) + r.GET("/t", func(c *gin.Context) { c.String(200, GetSecurityClientIP(c, false)) }) w := httptest.NewRecorder() req := httptest.NewRequest("GET", "/t", nil) @@ -117,3 +106,54 @@ func TestGetSecurityClientIPUsesConfiguredTrustedProxy(t *testing.T) { require.Equal(t, "1.2.3.4", w.Body.String()) } + +func TestGetClientIPSwitchDisabledUsesTrustedProxyChain(t *testing.T) { + gin.SetMode(gin.TestMode) + r := gin.New() + require.NoError(t, r.SetTrustedProxies(nil)) + r.GET("/t", func(c *gin.Context) { + SetLegacyForwardedIPTrust(c, false) + c.String(200, GetClientIP(c)) + }) + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/t", nil) + req.RemoteAddr = "9.9.9.9:12345" + req.Header.Set("X-Real-IP", "1.2.3.4") + r.ServeHTTP(w, req) + + require.Equal(t, "9.9.9.9", w.Body.String()) +} + +func TestGetSecurityClientIPRequestSnapshotOverridesLiveFallback(t *testing.T) { + gin.SetMode(gin.TestMode) + + tests := []struct { + name string + requestTrust bool + fallbackTrust bool + want string + }{ + {name: "captured secure mode wins", requestTrust: false, fallbackTrust: true, want: "9.9.9.9"}, + {name: "captured compatibility mode wins", requestTrust: true, fallbackTrust: false, want: "1.2.3.4"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + r := gin.New() + require.NoError(t, r.SetTrustedProxies(nil)) + r.GET("/t", func(c *gin.Context) { + SetLegacyForwardedIPTrust(c, test.requestTrust) + c.String(200, GetSecurityClientIP(c, test.fallbackTrust)) + }) + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/t", nil) + req.RemoteAddr = "9.9.9.9:12345" + req.Header.Set("X-Real-IP", "1.2.3.4") + r.ServeHTTP(w, req) + + require.Equal(t, test.want, w.Body.String()) + }) + } +} From 39107ca4570c05e5779f49e127d819d1d1dce7a0 Mon Sep 17 00:00:00 2001 From: Jlypx Date: Sun, 19 Jul 2026 21:40:32 +0800 Subject: [PATCH 32/59] =?UTF-8?q?fix:=20=E5=AE=89=E5=85=A8=E5=88=9D?= =?UTF-8?q?=E5=A7=8B=E5=8C=96=20Gin=20=E5=8F=AF=E4=BF=A1=E4=BB=A3=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- backend/internal/server/http.go | 32 ++++++++----- backend/internal/server/http_ingress_test.go | 50 ++++++++++++++++++++ 2 files changed, 70 insertions(+), 12 deletions(-) diff --git a/backend/internal/server/http.go b/backend/internal/server/http.go index 65548c9e24..fef7cd056e 100644 --- a/backend/internal/server/http.go +++ b/backend/internal/server/http.go @@ -47,18 +47,7 @@ func ProvideRouter( r := gin.New() r.Use(middleware2.Recovery()) - if len(cfg.Server.TrustedProxies) > 0 { - if err := r.SetTrustedProxies(cfg.Server.TrustedProxies); err != nil { - log.Printf("Failed to set trusted proxies: %v", err) - } - } else { - if err := r.SetTrustedProxies(nil); err != nil { - log.Printf("Failed to disable trusted proxies: %v", err) - } - if cfg.Server.Mode == "release" { - log.Printf("Warning: server.trusted_proxies is empty in release mode; client IP trust chain is disabled") - } - } + configureTrustedProxies(r, cfg.Server) // Wire up websearch Manager builder so it initializes on startup and rebuilds on config save. settingService.SetWebSearchManagerBuilder(context.Background(), func(cfg *service.WebSearchEmulationConfig, proxyURLs map[int64]string) { @@ -99,6 +88,25 @@ func ProvideRouter( return SetupRouter(r, handlers, jwtAuth, adminAuth, apiKeyAuth, auditLog, stepUpAuth, apiKeyService, subscriptionService, opsService, settingService, cfg, redisClient) } +func configureTrustedProxies(r *gin.Engine, cfg config.ServerConfig) { + if cfg.TrustedProxiesConfigured { + if err := r.SetTrustedProxies(cfg.TrustedProxies); err != nil { + log.Printf("Failed to set trusted proxies: %v", err) + _ = r.SetTrustedProxies(nil) + } + if len(cfg.TrustedProxies) == 0 && cfg.Mode == "release" { + log.Printf("Warning: server.trusted_proxies is explicitly empty; forwarded client IP trust is disabled") + } + } else { + if err := r.SetTrustedProxies(nil); err != nil { + log.Printf("Failed to disable trusted proxies: %v", err) + } + if cfg.Mode == "release" { + log.Printf("Warning: server.trusted_proxies is not configured; disabling the forwarded-IP compatibility switch will use direct peer addresses only") + } + } +} + // ProvideHTTPServer 提供 HTTP 服务器 func ProvideHTTPServer(cfg *config.Config, router *gin.Engine) *http.Server { httpHandler := http.Handler(router) diff --git a/backend/internal/server/http_ingress_test.go b/backend/internal/server/http_ingress_test.go index 63cf7a047a..5e986bbd7c 100644 --- a/backend/internal/server/http_ingress_test.go +++ b/backend/internal/server/http_ingress_test.go @@ -55,6 +55,56 @@ func TestProvideHTTPServerEnablesBoundedH2C(t *testing.T) { require.True(t, srv.Protocols.HTTP1()) } +func TestConfigureTrustedProxies(t *testing.T) { + gin.SetMode(gin.TestMode) + tests := []struct { + name string + cfg config.ServerConfig + want string + }{ + { + name: "configured proxy resolves forwarded client", + cfg: config.ServerConfig{ + TrustedProxies: []string{"9.9.9.9/32"}, + TrustedProxiesConfigured: true, + }, + want: "1.2.3.4", + }, + { + name: "explicit empty list ignores forwarded client", + cfg: config.ServerConfig{ + TrustedProxiesConfigured: true, + }, + want: "9.9.9.9", + }, + { + name: "invalid proxy list fails closed", + cfg: config.ServerConfig{ + TrustedProxies: []string{"not-a-cidr"}, + TrustedProxiesConfigured: true, + }, + want: "9.9.9.9", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + r := gin.New() + configureTrustedProxies(r, tc.cfg) + r.GET("/t", func(c *gin.Context) { c.String(http.StatusOK, c.ClientIP()) }) + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/t", nil) + req.RemoteAddr = "9.9.9.9:12345" + req.Header.Set("X-Forwarded-For", "1.2.3.4") + r.ServeHTTP(w, req) + + require.Equal(t, http.StatusOK, w.Code) + require.Equal(t, tc.want, w.Body.String()) + }) + } +} + func TestHTTPServerRejectsOversizedHTTP1Header(t *testing.T) { r := gin.New() r.GET("/", func(c *gin.Context) { c.Status(http.StatusOK) }) From 6aa6b3a96832d23298502d65104bcce7a0c78120 Mon Sep 17 00:00:00 2001 From: Jlypx Date: Sun, 19 Jul 2026 21:41:12 +0800 Subject: [PATCH 33/59] =?UTF-8?q?fix:=20=E5=B0=86=E5=AE=A2=E6=88=B7?= =?UTF-8?q?=E7=AB=AF=20IP=20=E6=A8=A1=E5=BC=8F=E6=B3=A8=E5=85=A5=E8=AF=B7?= =?UTF-8?q?=E6=B1=82=E4=B8=8A=E4=B8=8B=E6=96=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .../internal/server/middleware/api_key_auth_test.go | 7 ++++--- .../internal/server/middleware/session_binding.go | 13 +++++++------ .../server/middleware/session_binding_test.go | 10 ++++++---- backend/internal/server/router.go | 2 +- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/backend/internal/server/middleware/api_key_auth_test.go b/backend/internal/server/middleware/api_key_auth_test.go index a3216f8c22..4d6d9b5240 100644 --- a/backend/internal/server/middleware/api_key_auth_test.go +++ b/backend/internal/server/middleware/api_key_auth_test.go @@ -863,7 +863,7 @@ func TestRequireGroupAssignmentMarksUngroupedKeyBusinessLimited(t *testing.T) { require.Equal(t, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnassigned, businessLimitedReason) } -func TestAPIKeyAuthIPRestrictionDoesNotTrustForwardedClientIPByDefault(t *testing.T) { +func TestAPIKeyAuthIPRestrictionUsesTrustedPathWhenSwitchDisabled(t *testing.T) { gin.SetMode(gin.TestMode) user := &service.User{ @@ -893,6 +893,7 @@ func TestAPIKeyAuthIPRestrictionDoesNotTrustForwardedClientIPByDefault(t *testin } cfg := &config.Config{RunMode: config.RunModeSimple} + cfg.SetTrustForwardedIPForAPIKeyACL(false) apiKeyService := service.NewAPIKeyService(apiKeyRepo, nil, nil, nil, nil, nil, cfg) router := gin.New() require.NoError(t, router.SetTrustedProxies(nil)) @@ -1003,7 +1004,7 @@ func TestAPIKeyAuthIPRestrictionUsesConfiguredTrustedProxy(t *testing.T) { } cfg := &config.Config{RunMode: config.RunModeSimple} - cfg.SetTrustForwardedIPForAPIKeyACL(true) + cfg.SetTrustForwardedIPForAPIKeyACL(false) apiKeyService := service.NewAPIKeyService(apiKeyRepo, nil, nil, nil, nil, nil, cfg) router := gin.New() require.NoError(t, router.SetTrustedProxies([]string{"9.9.9.9"})) @@ -1054,7 +1055,7 @@ func TestAPIKeyAuthIPRestrictionUsesForwardedClientIPInDenialWhenTrusted(t *test } cfg := &config.Config{RunMode: config.RunModeSimple} - cfg.SetTrustForwardedIPForAPIKeyACL(true) + cfg.SetTrustForwardedIPForAPIKeyACL(false) apiKeyService := service.NewAPIKeyService(apiKeyRepo, nil, nil, nil, nil, nil, cfg) router := gin.New() require.NoError(t, router.SetTrustedProxies([]string{"9.9.9.9"})) diff --git a/backend/internal/server/middleware/session_binding.go b/backend/internal/server/middleware/session_binding.go index e535693c27..3f4d04b935 100644 --- a/backend/internal/server/middleware/session_binding.go +++ b/backend/internal/server/middleware/session_binding.go @@ -13,14 +13,16 @@ import ( // SessionBindingContext 全局中间件：将请求的客户端 IP 与 User-Agent 注入 // request context，供 token 签发路径（登录 / 刷新 / OAuth 回调）读取并写入会话绑定， // 同时作为审计日志、会话绑定校验的统一客户端 IP 来源。 -// IP 取值与 API Key IP 限制共用 Gin trusted_proxies 解析链；旧设置开关 -// 仅为配置兼容保留，不能单独使直连请求的转发头变为可信。 +// IP 取值与 API Key IP 限制共用转发 IP 开关：开启时旧版原始转发头逻辑 +// 接管解析，关闭时使用 Gin 的 server.trusted_proxies 可信代理链。 func SessionBindingContext(cfg *config.Config) gin.HandlerFunc { return func(c *gin.Context) { + trustForwarded := cfg.TrustForwardedIPForAPIKeyACL() + ip.SetLegacyForwardedIPTrust(c, trustForwarded) userAgent := normalizePersistentText(c.Request.UserAgent(), maxPersistentUserAgentBytes) c.Request.Header.Set("User-Agent", userAgent) binding := &service.SessionBinding{ - IP: ip.GetSecurityClientIP(c, cfg.TrustForwardedIPForAPIKeyACL()), + IP: ip.GetSecurityClientIP(c, trustForwarded), UserAgent: userAgent, } c.Request = c.Request.WithContext(service.WithSessionBinding(c.Request.Context(), binding)) @@ -29,8 +31,7 @@ func SessionBindingContext(cfg *config.Config) gin.HandlerFunc { } // requestSessionBinding 返回当前请求的会话指纹，优先取 SessionBindingContext -// 注入的解析结果（保证与 token 签发路径取值一致）；注入缺失时按 trusted_proxies -// 链回退兜底（等价于开关关闭时的行为）。 +// 注入的解析结果（保证与 token 签发路径取值一致）；注入缺失时使用安全回退。 func requestSessionBinding(c *gin.Context) *service.SessionBinding { if binding := service.SessionBindingFromContext(c.Request.Context()); binding != nil { return binding @@ -42,7 +43,7 @@ func requestSessionBinding(c *gin.Context) *service.SessionBinding { } // SecurityClientIP 返回当前请求用于安全敏感记录（审计日志等）的客户端 IP。 -// 与会话绑定、API Key IP 限制共用同一套「信任反代传递的客户端 IP」开关语义。 +// 与会话绑定、API Key IP 限制共用同一套客户端 IP 来源。 func SecurityClientIP(c *gin.Context) string { if binding := service.SessionBindingFromContext(c.Request.Context()); binding != nil && strings.TrimSpace(binding.IP) != "" { diff --git a/backend/internal/server/middleware/session_binding_test.go b/backend/internal/server/middleware/session_binding_test.go index 839983623d..11a098d9fd 100644 --- a/backend/internal/server/middleware/session_binding_test.go +++ b/backend/internal/server/middleware/session_binding_test.go @@ -14,23 +14,25 @@ import ( "github.com/stretchr/testify/require" ) -func TestSessionBindingContextDoesNotTrustHeadersWithoutTrustedProxy(t *testing.T) { +func TestSessionBindingContextFollowsForwardedIPSwitch(t *testing.T) { gin.SetMode(gin.TestMode) for _, tc := range []struct { name string trustForwarded bool + trustedProxies []string wantIP string }{ - {name: "trust disabled records proxy address", trustForwarded: false, wantIP: "127.0.0.1"}, - {name: "legacy trust toggle cannot bypass trusted proxies", trustForwarded: true, wantIP: "127.0.0.1"}, + {name: "enabled switch takes over raw headers", trustForwarded: true, wantIP: "1.2.3.4"}, + {name: "disabled switch ignores untrusted headers", trustForwarded: false, wantIP: "127.0.0.1"}, + {name: "disabled switch uses configured Gin proxy", trustForwarded: false, trustedProxies: []string{"127.0.0.1"}, wantIP: "1.2.3.4"}, } { t.Run(tc.name, func(t *testing.T) { cfg := &config.Config{} cfg.SetTrustForwardedIPForAPIKeyACL(tc.trustForwarded) r := gin.New() - require.NoError(t, r.SetTrustedProxies(nil)) + require.NoError(t, r.SetTrustedProxies(tc.trustedProxies)) r.Use(SessionBindingContext(cfg)) r.GET("/t", func(c *gin.Context) { binding := service.SessionBindingFromContext(c.Request.Context()) diff --git a/backend/internal/server/router.go b/backend/internal/server/router.go index 47bffbef22..e10672d83e 100644 --- a/backend/internal/server/router.go +++ b/backend/internal/server/router.go @@ -56,7 +56,7 @@ func SetupRouter( // 应用中间件 r.Use(middleware2.RequestLogger()) // 将客户端 IP + UA 注入 request context，供 token 签发/会话绑定/审计日志统一读取。 - // IP 取值与 API Key IP 限制共用 server.trusted_proxies 信任链。 + // 解析模式按请求快照：兼容开关开启时信任原始转发头，关闭时使用 server.trusted_proxies。 r.Use(middleware2.SessionBindingContext(cfg)) r.Use(middleware2.Logger()) r.Use(middleware2.CORS(cfg.CORS)) From 93717394b22b4ad2bd222e478733de2ef5ef7d74 Mon Sep 17 00:00:00 2001 From: Jlypx Date: Sun, 19 Jul 2026 21:41:45 +0800 Subject: [PATCH 34/59] =?UTF-8?q?fix:=20=E8=BF=81=E7=A7=BB=E5=AE=A2?= =?UTF-8?q?=E6=88=B7=E7=AB=AF=20IP=20=E5=85=BC=E5=AE=B9=E5=BC=80=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- backend/internal/service/domain_constants.go | 1 + backend/internal/service/setting_parse.go | 5 +- backend/internal/service/setting_service.go | 36 +++- .../service/setting_service_update_test.go | 164 ++++++++++++++++++ 4 files changed, 196 insertions(+), 10 deletions(-) diff --git a/backend/internal/service/domain_constants.go b/backend/internal/service/domain_constants.go index cbbb4b7850..c99d751c87 100644 --- a/backend/internal/service/domain_constants.go +++ b/backend/internal/service/domain_constants.go @@ -165,6 +165,7 @@ const ( // API Key IP 访问控制设置 SettingKeyAPIKeyACLTrustForwardedIP = "api_key_acl_trust_forwarded_ip" // API Key IP 白/黑名单是否信任转发 IP + settingKeyForwardedClientIPModeV2 = "forwarded_client_ip_mode_v2_migrated" // TOTP 双因素认证设置 SettingKeyTotpEnabled = "totp_enabled" // 是否启用 TOTP 2FA 功能 diff --git a/backend/internal/service/setting_parse.go b/backend/internal/service/setting_parse.go index a56e57784e..4481bf262d 100644 --- a/backend/internal/service/setting_parse.go +++ b/backend/internal/service/setting_parse.go @@ -54,7 +54,8 @@ func (s *SettingService) InitializeDefaultSettings(ctx context.Context) error { SettingKeyLoginAgreementMode: defaultLoginAgreementMode, SettingKeyLoginAgreementUpdatedAt: defaultLoginAgreementDate, SettingKeyLoginAgreementDocuments: loginAgreementDocumentsJSON, - SettingKeyAPIKeyACLTrustForwardedIP: "false", + SettingKeyAPIKeyACLTrustForwardedIP: "true", + settingKeyForwardedClientIPModeV2: "true", SettingKeySiteName: "Sub2API", SettingKeySiteLogo: "", SettingKeyPurchaseSubscriptionEnabled: "false", @@ -249,7 +250,7 @@ func (s *SettingService) parseSettings(settings map[string]string) *SystemSettin if value, ok := settings[SettingKeyAPIKeyACLTrustForwardedIP]; ok { apiKeyACLTrustForwardedIP = value == "true" } else if s != nil && s.cfg != nil { - apiKeyACLTrustForwardedIP = s.cfg.Security.TrustForwardedIPForAPIKeyACL + apiKeyACLTrustForwardedIP = s.cfg.ForwardedClientIPTrustEnabled() } result := &SystemSettings{ RegistrationEnabled: settings[SettingKeyRegistrationEnabled] == "true", diff --git a/backend/internal/service/setting_service.go b/backend/internal/service/setting_service.go index 52acc62cf9..608cc0320b 100644 --- a/backend/internal/service/setting_service.go +++ b/backend/internal/service/setting_service.go @@ -2,7 +2,6 @@ package service import ( "context" - "errors" "fmt" "sync/atomic" @@ -220,15 +219,36 @@ func (s *SettingService) LoadAPIKeyACLTrustForwardedIPSetting(ctx context.Contex if s == nil || s.cfg == nil || s.settingRepo == nil { return nil } - value, err := s.settingRepo.GetValue(ctx, SettingKeyAPIKeyACLTrustForwardedIP) + + values, err := s.settingRepo.GetMultiple(ctx, []string{ + SettingKeyAPIKeyACLTrustForwardedIP, + settingKeyForwardedClientIPModeV2, + }) if err != nil { - if errors.Is(err, ErrSettingNotFound) { - s.cfg.SetTrustForwardedIPForAPIKeyACL(s.cfg.Security.TrustForwardedIPForAPIKeyACL) - return nil - } - return fmt.Errorf("get api key acl forwarded ip setting: %w", err) + s.cfg.SetTrustForwardedIPForAPIKeyACL(false) + return fmt.Errorf("get forwarded client ip settings: %w", err) } - enabled := value == "true" + + enabled := s.cfg.Security.TrustForwardedIPForAPIKeyACL + storedValue, hasStoredValue := values[SettingKeyAPIKeyACLTrustForwardedIP] + if hasStoredValue { + enabled = storedValue == "true" + } + + if values[settingKeyForwardedClientIPModeV2] != "true" { + updates := map[string]string{settingKeyForwardedClientIPModeV2: "true"} + // Before this migration, new installations persisted false by default. + // Restore compatibility only when no trusted-proxy policy was configured. + if hasStoredValue && !enabled && !s.cfg.Server.TrustedProxiesConfigured { + enabled = true + updates[SettingKeyAPIKeyACLTrustForwardedIP] = "true" + } + if err := s.settingRepo.SetMultiple(ctx, updates); err != nil { + s.cfg.SetTrustForwardedIPForAPIKeyACL(enabled) + return fmt.Errorf("migrate forwarded client ip setting: %w", err) + } + } + s.cfg.SetTrustForwardedIPForAPIKeyACL(enabled) return nil } diff --git a/backend/internal/service/setting_service_update_test.go b/backend/internal/service/setting_service_update_test.go index 93cf8284ad..efbf646feb 100644 --- a/backend/internal/service/setting_service_update_test.go +++ b/backend/internal/service/setting_service_update_test.go @@ -5,6 +5,7 @@ package service import ( "context" "encoding/json" + "errors" "math" "strconv" "testing" @@ -87,6 +88,58 @@ func (s *settingGetAllRepoStub) Delete(ctx context.Context, key string) error { panic("unexpected Delete call") } +type forwardedIPMigrationRepoStub struct { + values map[string]string + updates map[string]string + getMultipleErr error + setMultipleErr error +} + +func (s *forwardedIPMigrationRepoStub) Get(context.Context, string) (*Setting, error) { + panic("unexpected Get call") +} + +func (s *forwardedIPMigrationRepoStub) GetValue(context.Context, string) (string, error) { + panic("unexpected GetValue call") +} + +func (s *forwardedIPMigrationRepoStub) Set(context.Context, string, string) error { + panic("unexpected Set call") +} + +func (s *forwardedIPMigrationRepoStub) GetMultiple(_ context.Context, keys []string) (map[string]string, error) { + if s.getMultipleErr != nil { + return nil, s.getMultipleErr + } + result := make(map[string]string, len(keys)) + for _, key := range keys { + if value, ok := s.values[key]; ok { + result[key] = value + } + } + return result, nil +} + +func (s *forwardedIPMigrationRepoStub) SetMultiple(_ context.Context, values map[string]string) error { + if s.setMultipleErr != nil { + return s.setMultipleErr + } + s.updates = make(map[string]string, len(values)) + for key, value := range values { + s.values[key] = value + s.updates[key] = value + } + return nil +} + +func (s *forwardedIPMigrationRepoStub) GetAll(context.Context) (map[string]string, error) { + panic("unexpected GetAll call") +} + +func (s *forwardedIPMigrationRepoStub) Delete(context.Context, string) error { + panic("unexpected Delete call") +} + type settingAntigravityUARepoStub struct { values map[string]string } @@ -519,6 +572,117 @@ func TestSettingService_ParseSettings_APIKeyACLTrustForwardedIPFallsBackToConfig require.True(t, got.APIKeyACLTrustForwardedIP) } +func TestSettingService_ParseSettings_APIKeyACLTrustForwardedIPUsesStoredValue(t *testing.T) { + cfg := &config.Config{} + cfg.SetTrustForwardedIPForAPIKeyACL(true) + svc := NewSettingService(&settingUpdateRepoStub{}, cfg) + + got := svc.parseSettings(map[string]string{SettingKeyAPIKeyACLTrustForwardedIP: "false"}) + + require.False(t, got.APIKeyACLTrustForwardedIP) +} + +func TestSettingService_LoadAPIKeyACLTrustForwardedIPSettingMigration(t *testing.T) { + tests := []struct { + name string + values map[string]string + trustedProxiesSet bool + configDefault bool + wantEnabled bool + wantForwardedIPUpdate string + wantMigrationMarkerSet bool + }{ + { + name: "missing setting follows configured default", + values: map[string]string{}, + configDefault: true, + wantEnabled: true, + wantMigrationMarkerSet: true, + }, + { + name: "legacy false without proxy config migrates to compatibility", + values: map[string]string{SettingKeyAPIKeyACLTrustForwardedIP: "false"}, + wantEnabled: true, + wantForwardedIPUpdate: "true", + wantMigrationMarkerSet: true, + }, + { + name: "legacy false with explicit proxy config stays secure", + values: map[string]string{SettingKeyAPIKeyACLTrustForwardedIP: "false"}, + trustedProxiesSet: true, + wantEnabled: false, + wantMigrationMarkerSet: true, + }, + { + name: "completed migration preserves later false choice", + values: map[string]string{ + SettingKeyAPIKeyACLTrustForwardedIP: "false", + settingKeyForwardedClientIPModeV2: "true", + }, + wantEnabled: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + repo := &forwardedIPMigrationRepoStub{values: test.values} + cfg := &config.Config{Server: config.ServerConfig{TrustedProxiesConfigured: test.trustedProxiesSet}} + cfg.Security.TrustForwardedIPForAPIKeyACL = test.configDefault + svc := NewSettingService(repo, cfg) + + require.NoError(t, svc.LoadAPIKeyACLTrustForwardedIPSetting(context.Background())) + require.Equal(t, test.wantEnabled, cfg.TrustForwardedIPForAPIKeyACL()) + require.Equal(t, test.wantForwardedIPUpdate, repo.updates[SettingKeyAPIKeyACLTrustForwardedIP]) + if test.wantMigrationMarkerSet { + require.Equal(t, "true", repo.updates[settingKeyForwardedClientIPModeV2]) + } else { + require.Nil(t, repo.updates) + } + }) + } +} + +func TestSettingService_LoadAPIKeyACLTrustForwardedIPSettingReadFailureFailsClosed(t *testing.T) { + repo := &forwardedIPMigrationRepoStub{ + getMultipleErr: errors.New("database unavailable"), + } + cfg := &config.Config{} + cfg.SetTrustForwardedIPForAPIKeyACL(true) + svc := NewSettingService(repo, cfg) + + err := svc.LoadAPIKeyACLTrustForwardedIPSetting(context.Background()) + + require.ErrorContains(t, err, "get forwarded client ip settings") + require.False(t, cfg.TrustForwardedIPForAPIKeyACL()) +} + +func TestSettingService_LoadAPIKeyACLTrustForwardedIPSettingWriteFailureUsesComputedMode(t *testing.T) { + tests := []struct { + name string + trustedProxiesSet bool + wantEnabled bool + }{ + {name: "compatibility migration remains effective", wantEnabled: true}, + {name: "explicit proxy policy remains secure", trustedProxiesSet: true, wantEnabled: false}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + repo := &forwardedIPMigrationRepoStub{ + values: map[string]string{SettingKeyAPIKeyACLTrustForwardedIP: "false"}, + setMultipleErr: errors.New("database unavailable"), + } + cfg := &config.Config{Server: config.ServerConfig{TrustedProxiesConfigured: test.trustedProxiesSet}} + svc := NewSettingService(repo, cfg) + + err := svc.LoadAPIKeyACLTrustForwardedIPSetting(context.Background()) + + require.ErrorContains(t, err, "migrate forwarded client ip setting") + require.Equal(t, test.wantEnabled, cfg.TrustForwardedIPForAPIKeyACL()) + }) + } +} + func TestSettingService_GetAntigravityUserAgentVersion_Precedence(t *testing.T) { t.Run("后台设置优先", func(t *testing.T) { svc := NewSettingService(&settingAntigravityUARepoStub{values: map[string]string{ From 5da3325a306dd45afe5bd7178e03fa829310ca3d Mon Sep 17 00:00:00 2001 From: Jlypx Date: Sun, 19 Jul 2026 21:42:12 +0800 Subject: [PATCH 35/59] =?UTF-8?q?feat:=20=E5=9C=A8=E5=AE=89=E5=85=A8?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E4=B8=AD=E9=85=8D=E7=BD=AE=E5=AE=A2=E6=88=B7?= =?UTF-8?q?=E7=AB=AF=20IP=20=E6=A8=A1=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .../src/i18n/locales/en/admin/settings.ts | 2 +- .../src/i18n/locales/zh/admin/settings.ts | 2 +- frontend/src/views/admin/SettingsView.vue | 2 +- .../admin/__tests__/SettingsView.spec.ts | 29 +++++++++++++++++++ 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/frontend/src/i18n/locales/en/admin/settings.ts b/frontend/src/i18n/locales/en/admin/settings.ts index 531769eec1..c08ca2ae52 100644 --- a/frontend/src/i18n/locales/en/admin/settings.ts +++ b/frontend/src/i18n/locales/en/admin/settings.ts @@ -153,7 +153,7 @@ export default { 'Choose which client IP is used by API Key allowlists/denylists, admin audit logs, and session IP/UA binding', trustForwardedIp: 'Trust forwarded client IP', trustForwardedIpHint: - 'Disabled by default. Enable only when the origin is reachable only through Cloudflare or Nginx reverse proxy. When enabled, API Key IP allowlists/denylists, admin audit logs, and session IP/UA binding use CF-Connecting-IP, X-Real-IP, or X-Forwarded-For, matching the request IP shown in usage records. Toggling this switch changes the IP fingerprint of existing sessions; with session binding enabled they must sign in again.' + 'Enabled by default for upgrade compatibility. When enabled, raw CF-Connecting-IP, X-Real-IP, or X-Forwarded-For values take over server.trusted_proxies for client-IP resolution. Disable it to enforce the Gin trusted-proxy chain configured by server.trusted_proxies. Only enable takeover mode when the origin cannot be reached directly. Changing this switch changes existing session IP fingerprints.' }, linuxdo: { title: 'LinuxDo Connect Login', diff --git a/frontend/src/i18n/locales/zh/admin/settings.ts b/frontend/src/i18n/locales/zh/admin/settings.ts index f656ac4886..bfd3f139d5 100644 --- a/frontend/src/i18n/locales/zh/admin/settings.ts +++ b/frontend/src/i18n/locales/zh/admin/settings.ts @@ -152,7 +152,7 @@ export default { description: '控制 API Key 白/黑名单、操作审计日志与会话 IP/UA 绑定使用哪个客户端 IP 判断', trustForwardedIp: '信任反代传递的客户端 IP', trustForwardedIpHint: - '默认关闭。仅在源站只允许 Cloudflare 或 Nginx 反代访问时开启；开启后 API Key IP 白/黑名单、操作审计日志与会话 IP/UA 绑定会使用 CF-Connecting-IP、X-Real-IP 或 X-Forwarded-For，与使用记录中的请求 IP 保持一致。切换本开关会改变已登录会话的 IP 指纹，开启会话绑定时现有会话需重新登录。' + '为保证升级兼容默认开启。开启后 CF-Connecting-IP、X-Real-IP 或 X-Forwarded-For 会直接接管客户端 IP 解析并覆盖 server.trusted_proxies；关闭后严格使用 server.trusted_proxies 配置的 Gin 可信代理链。仅在源站无法被直接访问时开启接管模式。切换会改变现有会话的 IP 指纹。' }, linuxdo: { title: 'LinuxDo Connect 登录', diff --git a/frontend/src/views/admin/SettingsView.vue b/frontend/src/views/admin/SettingsView.vue index b2b1c760c8..c5c898ee85 100644 --- a/frontend/src/views/admin/SettingsView.vue +++ b/frontend/src/views/admin/SettingsView.vue @@ -8393,7 +8393,7 @@ const form = reactive({ turnstile_site_key: "", turnstile_secret_key: "", turnstile_secret_key_configured: false, - api_key_acl_trust_forwarded_ip: false, + api_key_acl_trust_forwarded_ip: true, // LinuxDo Connect OAuth 登录 linuxdo_connect_enabled: false, linuxdo_connect_client_id: "", diff --git a/frontend/src/views/admin/__tests__/SettingsView.spec.ts b/frontend/src/views/admin/__tests__/SettingsView.spec.ts index 8f73963362..8b7ffb301c 100644 --- a/frontend/src/views/admin/__tests__/SettingsView.spec.ts +++ b/frontend/src/views/admin/__tests__/SettingsView.spec.ts @@ -367,6 +367,7 @@ const baseSettingsResponse = { turnstile_enabled: false, turnstile_site_key: "", turnstile_secret_key_configured: false, + api_key_acl_trust_forwarded_ip: true, linuxdo_connect_enabled: false, linuxdo_connect_client_id: "", linuxdo_connect_client_secret_configured: false, @@ -652,6 +653,34 @@ describe("admin SettingsView payment visible method controls", () => { expect(wrapper.text()).not.toContain("支付来源"); }); + it("loads and saves the forwarded client IP takeover switch", async () => { + getSettings.mockResolvedValueOnce({ + ...baseSettingsResponse, + api_key_acl_trust_forwarded_ip: false, + }); + const wrapper = mountView(); + + await flushPromises(); + await openSecurityTab(wrapper); + + const card = wrapper + .findAll(".card") + .find((node) => node.text().includes("admin.settings.apiKeyAcl.title")); + expect(card).toBeDefined(); + const toggle = card!.get('input[type="checkbox"]'); + expect((toggle.element as HTMLInputElement).checked).toBe(false); + + await toggle.setValue(true); + await wrapper.find("form").trigger("submit.prevent"); + await flushPromises(); + + expect(updateSettings).toHaveBeenCalledWith( + expect.objectContaining({ + api_key_acl_trust_forwarded_ip: true, + }), + ); + }); + it("links payment guidance to README sections instead of removed payment docs", async () => { const wrapper = mountView(); From 41b58b640a607a6593ee1bafce782c3f6de4e09a Mon Sep 17 00:00:00 2001 From: Jlypx Date: Sun, 19 Jul 2026 21:42:33 +0800 Subject: [PATCH 36/59] =?UTF-8?q?docs:=20=E6=9B=B4=E6=96=B0=E5=8F=AF?= =?UTF-8?q?=E4=BF=A1=E4=BB=A3=E7=90=86=E9=83=A8=E7=BD=B2=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- deploy/EDGE_SECURITY.md | 28 +++++++++++++++------------- deploy/config.example.yaml | 22 ++++++++++++---------- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/deploy/EDGE_SECURITY.md b/deploy/EDGE_SECURITY.md index b2c08cf20c..0c0e78768a 100644 --- a/deploy/EDGE_SECURITY.md +++ b/deploy/EDGE_SECURITY.md @@ -29,21 +29,23 @@ the application's responsibility. ## Trusted client IPs -`server.trusted_proxies` controls forwarded-IP trust for security-sensitive -paths such as API-key ACLs, session binding, and rejection aggregation. Fresh -installations default to local/container ranges (`127.0.0.0/8`, `::1/128`, -`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, and `fc00::/7`) so a local -Nginx/Caddy or Docker bridge works without a migration. For a remote load -balancer, replace the defaults with only the CIDRs that connect directly to -Sub2API. An explicit empty list disables forwarded-IP trust for these paths; -ordinary request/usage metadata keeps its legacy compatibility behavior. +`security.trust_forwarded_ip_for_api_key_acl` is enabled by default for upgrade +compatibility. On the first upgrade to this mode, a legacy `false` value is +changed to `true` only when `server.trusted_proxies` was not explicitly +configured; explicit proxy policies remain in secure mode. Later administrator +changes are preserved. While enabled, raw `CF-Connecting-IP`, `X-Real-IP`, and +`X-Forwarded-For` values take over client-IP resolution for logs and +security-sensitive paths. Disable the switch to make Gin's +`server.trusted_proxies` chain authoritative. Configure only the exact CIDR/IP +addresses that connect directly to Sub2API; an explicit empty list trusts no +forwarded client IPs while the switch is disabled. -Never use `CF-Connecting-IP`, `X-Real-IP`, or `X-Forwarded-For` for an ACL or -session decision merely because the header exists. A CDN deployment must -firewall the origin so only the CDN or load balancer can reach it, and the proxy -must overwrite forwarded headers. +Compatibility takeover accepts forwarded headers without validating the direct +peer. Protect the origin from direct access while it is enabled. A CDN +deployment must firewall the origin so only the CDN or load balancer can reach +it, and the proxy must overwrite forwarded headers. -Example for a proxy on the same host (the default already covers this case): +Example for a proxy on the same host: ```yaml server: diff --git a/deploy/config.example.yaml b/deploy/config.example.yaml index 2720c2d298..e84a906d4d 100644 --- a/deploy/config.example.yaml +++ b/deploy/config.example.yaml @@ -36,18 +36,15 @@ server: # Keep-alive idle timeout in seconds. # Keep-Alive 空闲连接超时（秒）。 idle_timeout: 120 - # Trusted proxies for security-sensitive X-Forwarded-For parsing (CIDR/IP). - # These local/container ranges are the default; replace them with the exact - # proxy CIDRs for a remote load balancer. Set [] explicitly to disable trust. - # 安全敏感场景解析 X-Forwarded-For 的可信代理（CIDR/IP）。以下为本机/容器 - # 网段默认值；远程负载均衡请替换为实际 CIDR。显式设置 [] 可禁用代理信任。 + # Trusted proxies used when security.trust_forwarded_ip_for_api_key_acl is false. + # List only the exact proxy addresses that connect directly to Sub2API. + # Set [] explicitly to disable forwarded-IP trust in high-security mode. + # security.trust_forwarded_ip_for_api_key_acl=false 时使用的可信代理。 + # 只填写直接连接 Sub2API 的精确代理地址；显式设置 [] 可在高安全模式下 + # 禁用转发 IP 信任。 trusted_proxies: - - 127.0.0.0/8 + - 127.0.0.1/32 - ::1/128 - - 10.0.0.0/8 - - 172.16.0.0/12 - - 192.168.0.0/16 - - fc00::/7 # Global max request body size in bytes (default: 256MB) # 全局最大请求体大小（字节，默认 256MB） # Applies to all requests, especially important for h2c first request memory protection @@ -104,6 +101,11 @@ cors: # 安全配置 # ============================================================================= security: + # Legacy compatibility switch. When true, raw forwarded headers take over + # server.trusted_proxies. Set false to enforce the trusted proxy chain above. + # 旧版兼容开关。开启时原始转发头会接管 server.trusted_proxies；关闭后严格 + # 使用上方配置的可信代理链。示例配置采用高安全模式。 + trust_forwarded_ip_for_api_key_acl: false url_allowlist: # Enable URL allowlist validation (disable to skip all URL checks) # 启用 URL 白名单验证（禁用则跳过所有 URL 检查） From f69042ca6e5541f2662311b451a6aca44d0a3f27 Mon Sep 17 00:00:00 2001 From: Jlypx Date: Sun, 19 Jul 2026 21:42:54 +0800 Subject: [PATCH 37/59] =?UTF-8?q?docs:=20=E6=9B=B4=E6=96=B0=E5=AE=A2?= =?UTF-8?q?=E6=88=B7=E7=AB=AF=20IP=20=E9=85=8D=E7=BD=AE=E7=B4=A2=E5=BC=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- README.md | 2 +- README_CN.md | 2 +- README_JA.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1e34840b00..16e7bd9a5d 100644 --- a/README.md +++ b/README.md @@ -571,7 +571,7 @@ Additional security-related options are available in `config.yaml`: - `security.response_headers.enabled` to enable configurable response header filtering (disabled uses default allowlist) - `security.csp` to control Content-Security-Policy headers - `billing.circuit_breaker` to fail closed on billing errors -- `server.trusted_proxies` to configure forwarded-IP trust for security-sensitive paths (local/container proxy ranges are trusted by default; replace them with exact remote proxy CIDRs when needed) +- `security.trust_forwarded_ip_for_api_key_acl` enables legacy raw forwarded-header takeover (enabled by default for upgrade compatibility); disable it to enforce `server.trusted_proxies`, which should contain only the exact proxy CIDRs that connect directly to Sub2API - `turnstile.required` to require Turnstile in release mode **⚠️ Security Warning: HTTP URL Configuration** diff --git a/README_CN.md b/README_CN.md index 632b6532db..a9aea3bbe8 100644 --- a/README_CN.md +++ b/README_CN.md @@ -607,7 +607,7 @@ gateway: - `security.response_headers.enabled` 可启用可配置响应头过滤（关闭时使用默认白名单） - `security.csp` 配置 Content-Security-Policy - `billing.circuit_breaker` 计费异常时 fail-closed -- `server.trusted_proxies` 配置安全敏感路径的反代 IP 信任（本机/常见 Docker 私网网段默认已信任；远程反代请替换为精确 CIDR） +- `security.trust_forwarded_ip_for_api_key_acl` 控制旧版原始转发头接管（为升级兼容默认开启）；关闭后严格使用 `server.trusted_proxies`，其中只应填写直接连接 Sub2API 的精确代理 CIDR - `turnstile.required` 在 release 模式强制启用 Turnstile **网关防御纵深建议（重点）** diff --git a/README_JA.md b/README_JA.md index 849d951f68..8d95162612 100644 --- a/README_JA.md +++ b/README_JA.md @@ -569,7 +569,7 @@ default: - `security.response_headers.enabled` - 設定可能なレスポンスヘッダーフィルタリングを有効化（無効時はデフォルトの許可リストを使用） - `security.csp` - Content-Security-Policy ヘッダーの制御 - `billing.circuit_breaker` - 課金エラー時にフェイルクローズ -- `server.trusted_proxies` - セキュリティ用途の転送 IP 信頼を設定（ローカル/一般的な Docker プライベート範囲はデフォルトで信頼。リモートプロキシは正確な CIDR に置換） +- `security.trust_forwarded_ip_for_api_key_acl` - 従来の生転送ヘッダーによる上書きを制御（アップグレード互換性のため既定で有効）。無効にすると `server.trusted_proxies` を厳格に使用し、Sub2API に直接接続するプロキシの正確な CIDR のみを指定 - `turnstile.required` - リリースモードでの Turnstile 必須化 **⚠️ セキュリティ警告: HTTP URL 設定** From 01062c324429636b36d349036fdf2d08e1dce699 Mon Sep 17 00:00:00 2001 From: wucm667 Date: Sun, 19 Jul 2026 22:28:49 +0800 Subject: [PATCH 38/59] fix(openai): correct buffered messages content type --- .../service/openai_compat_model_test.go | 38 +++++++++++++++++++ .../service/openai_gateway_messages.go | 1 + 2 files changed, 39 insertions(+) diff --git a/backend/internal/service/openai_compat_model_test.go b/backend/internal/service/openai_compat_model_test.go index e1007c507a..c7f82d7c3d 100644 --- a/backend/internal/service/openai_compat_model_test.go +++ b/backend/internal/service/openai_compat_model_test.go @@ -3,6 +3,7 @@ package service import ( "bytes" "context" + "encoding/json" "errors" "fmt" "io" @@ -1721,6 +1722,43 @@ func TestForwardAsAnthropic_BufferedTerminalWithoutUpstreamCloseReturns(t *testi } } +func TestHandleAnthropicBufferedStreamingResponse_OverridesUpstreamContentType(t *testing.T) { + gin.SetMode(gin.TestMode) + + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", nil) + + resp := &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"text/event-stream"}}, + Body: io.NopCloser(strings.NewReader( + `data: {"type":"response.completed","response":{"id":"resp_buffered_json","object":"response","model":"gpt-5.4","status":"completed","output":[{"type":"message","id":"msg_1","role":"assistant","status":"completed","content":[{"type":"output_text","text":"ok"}]}],"usage":{"input_tokens":15,"output_tokens":6,"total_tokens":21,"input_tokens_details":{"cached_tokens":5}}}}` + "\n\n", + )), + } + cfg := &config.Config{} + svc := &OpenAIGatewayService{ + cfg: cfg, + responseHeaderFilter: compileResponseHeaderFilter(cfg), + } + + result, err := svc.handleAnthropicBufferedStreamingResponse( + resp, c, &Account{}, "claude-sonnet-4-5", "gpt-5.4", "gpt-5.4", time.Now(), + ) + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "application/json; charset=utf-8", rec.Header().Get("Content-Type")) + require.NotContains(t, rec.Header().Get("Content-Type"), "text/event-stream") + + var message apicompat.AnthropicResponse + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &message)) + require.Equal(t, "message", message.Type) + require.Equal(t, "resp_buffered_json", message.ID) + require.Equal(t, 10, message.Usage.InputTokens) + require.Equal(t, 6, message.Usage.OutputTokens) + require.Equal(t, 5, message.Usage.CacheReadInputTokens) +} + func TestForwardAsAnthropic_BufferedEventNamedTerminalWithoutUpstreamCloseReturns(t *testing.T) { gin.SetMode(gin.TestMode) diff --git a/backend/internal/service/openai_gateway_messages.go b/backend/internal/service/openai_gateway_messages.go index f37f2997cd..db9e95a8fe 100644 --- a/backend/internal/service/openai_gateway_messages.go +++ b/backend/internal/service/openai_gateway_messages.go @@ -512,6 +512,7 @@ func (s *OpenAIGatewayService) handleAnthropicBufferedStreamingResponse( if s.responseHeaderFilter != nil { responseheaders.WriteFilteredHeaders(c.Writer.Header(), resp.Header, s.responseHeaderFilter) } + c.Header("Content-Type", "application/json; charset=utf-8") c.JSON(http.StatusOK, anthropicResp) return &OpenAIForwardResult{ From 35b5edb24ce0a030d41d3147f5159904f530bc90 Mon Sep 17 00:00:00 2001 From: superman2003 <2112076433zcr@gmail.com> Date: Sun, 19 Jul 2026 22:40:44 +0800 Subject: [PATCH 39/59] fix(update): detach in-place update from the HTTP request lifetime The in-place update ran entirely inside c.Request.Context(). Browsers and reverse proxies commonly abort long-idle requests (axios global timeout 30s, nginx proxy_read_timeout 60s by default), which canceled the request context mid-download and killed every slow update with 'download failed: context canceled' while the version stayed unchanged. Users behind slow GitHub links saw the update button fail at a wall-clock ceiling (~60s) on every attempt (#4504). - Run PerformUpdate and RollbackToVersion on a context detached from the request (context.WithoutCancel) and bounded by a 15-minute deadline so the 10-minute GitHub download client owns its own timeout. A client disconnect no longer aborts the binary swap; retries then hit the system operation lock or report 'Already up to date'. - Raise the frontend timeout for the update/rollback calls from the global 30s axios default to 15 minutes so the browser can actually wait for the result. Fixes #4504 --- .../internal/handler/admin/system_handler.go | 33 ++++++- .../handler/admin/system_handler_test.go | 85 ++++++++++++++++--- frontend/src/api/admin/system.ts | 15 +++- 3 files changed, 114 insertions(+), 19 deletions(-) diff --git a/backend/internal/handler/admin/system_handler.go b/backend/internal/handler/admin/system_handler.go index 49bd5bc20a..6afa29f05e 100644 --- a/backend/internal/handler/admin/system_handler.go +++ b/backend/internal/handler/admin/system_handler.go @@ -22,6 +22,27 @@ type SystemHandler struct { lockSvc *service.SystemOperationLockService } +// systemUpdateTimeout bounds a full in-place update or rollback: the release +// manifest fetch plus a large binary download over slow links. It must stay +// above the GitHub download client timeout (10 minutes) so the download owns +// its own deadline. +const systemUpdateTimeout = 15 * time.Minute + +// systemUpdateContext detaches a long-running update/rollback from the HTTP +// request lifetime. Browsers and reverse proxies commonly abort idle requests +// after 30-60s (axios default, nginx proxy_read_timeout), which canceled +// c.Request.Context() mid-download and killed the update with +// "download failed: context canceled" (#4504). The swap keeps running after a +// client disconnect; a later retry then hits the system operation lock or +// reports "Already up to date". +func systemUpdateContext(ctx context.Context) (context.Context, context.CancelFunc) { + base := context.Background() + if ctx != nil { + base = context.WithoutCancel(ctx) + } + return context.WithTimeout(base, systemUpdateTimeout) +} + type systemUpdateService interface { CheckUpdate(ctx context.Context, force bool) (*service.UpdateInfo, error) PerformUpdate(ctx context.Context) error @@ -75,9 +96,12 @@ func (h *SystemHandler) PerformUpdate(c *gin.Context) { release(releaseReason, succeeded) }() - if err := h.updateSvc.PerformUpdate(ctx); err != nil { + updateCtx, cancel := systemUpdateContext(ctx) + defer cancel() + + if err := h.updateSvc.PerformUpdate(updateCtx); err != nil { if errors.Is(err, service.ErrNoUpdateAvailable) { - info, checkErr := h.updateSvc.CheckUpdate(ctx, false) + info, checkErr := h.updateSvc.CheckUpdate(updateCtx, false) if checkErr != nil { releaseReason = "SYSTEM_UPDATE_FAILED" return nil, checkErr @@ -152,7 +176,10 @@ func (h *SystemHandler) Rollback(c *gin.Context) { }() if targetVersion != "" { - err = h.updateSvc.RollbackToVersion(ctx, targetVersion) + // 指定版本回退同样要下载完整二进制，与更新一样和请求生命周期解耦。 + rollbackCtx, cancel := systemUpdateContext(ctx) + defer cancel() + err = h.updateSvc.RollbackToVersion(rollbackCtx, targetVersion) } else { err = h.updateSvc.Rollback() } diff --git a/backend/internal/handler/admin/system_handler_test.go b/backend/internal/handler/admin/system_handler_test.go index 78e24d008e..fabee82d81 100644 --- a/backend/internal/handler/admin/system_handler_test.go +++ b/backend/internal/handler/admin/system_handler_test.go @@ -18,18 +18,22 @@ import ( ) type systemHandlerUpdateServiceStub struct { - performErr error - updateInfo *service.UpdateInfo - checkErr error - checkForces []bool - performCall int - rollbackCall int - rollbackToCall int - rollbackToVersions []string - rollbackToErr error - rollbackVersions []service.RollbackVersion - rollbackVersionsErr error - rollbackVersionsCall int + performErr error + updateInfo *service.UpdateInfo + checkErr error + checkForces []bool + performCall int + performCtxErr error + performHasDeadline bool + rollbackCall int + rollbackToCall int + rollbackToCtxErr error + rollbackToHasDeadline bool + rollbackToVersions []string + rollbackToErr error + rollbackVersions []service.RollbackVersion + rollbackVersionsErr error + rollbackVersionsCall int } func (s *systemHandlerUpdateServiceStub) CheckUpdate(_ context.Context, force bool) (*service.UpdateInfo, error) { @@ -37,8 +41,10 @@ func (s *systemHandlerUpdateServiceStub) CheckUpdate(_ context.Context, force bo return s.updateInfo, s.checkErr } -func (s *systemHandlerUpdateServiceStub) PerformUpdate(context.Context) error { +func (s *systemHandlerUpdateServiceStub) PerformUpdate(ctx context.Context) error { s.performCall++ + s.performCtxErr = ctx.Err() + _, s.performHasDeadline = ctx.Deadline() return s.performErr } @@ -52,8 +58,10 @@ func (s *systemHandlerUpdateServiceStub) ListRollbackVersions(context.Context) ( return s.rollbackVersions, s.rollbackVersionsErr } -func (s *systemHandlerUpdateServiceStub) RollbackToVersion(_ context.Context, version string) error { +func (s *systemHandlerUpdateServiceStub) RollbackToVersion(ctx context.Context, version string) error { s.rollbackToCall++ + s.rollbackToCtxErr = ctx.Err() + _, s.rollbackToHasDeadline = ctx.Deadline() s.rollbackToVersions = append(s.rollbackToVersions, version) return s.rollbackToErr } @@ -165,6 +173,55 @@ func TestSystemHandlerPerformUpdateFailureStillReturnsInternalError(t *testing.T require.Equal(t, "internal error", body.Message) } +// TestSystemHandlerPerformUpdateSurvivesClientDisconnect reproduces #4504: +// the browser or a reverse proxy (axios 30s default, nginx proxy_read_timeout +// 60s) aborts the long-running update request and cancels the request +// context. The download must keep running on a detached, bounded context +// instead of dying with "download failed: context canceled". +func TestSystemHandlerPerformUpdateSurvivesClientDisconnect(t *testing.T) { + updateSvc := &systemHandlerUpdateServiceStub{} + repo := newMemoryIdempotencyRepoStub() + router := newSystemHandlerTestRouter(t, updateSvc, repo) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/system/update", nil) + canceledCtx, cancel := context.WithCancel(context.Background()) + cancel() + req = req.WithContext(canceledCtx) + req.Header.Set("Idempotency-Key", "disconnected-update") + router.ServeHTTP(rec, req) + + require.Equal(t, 1, updateSvc.performCall) + require.NoError(t, updateSvc.performCtxErr, + "update must not observe the canceled request context") + require.True(t, updateSvc.performHasDeadline, + "detached update context must still be bounded by a deadline") + requireSystemLockStatus(t, repo, service.IdempotencyStatusSucceeded) +} + +func TestSystemHandlerRollbackToVersionSurvivesClientDisconnect(t *testing.T) { + updateSvc := &systemHandlerUpdateServiceStub{} + repo := newMemoryIdempotencyRepoStub() + router := newSystemHandlerTestRouter(t, updateSvc, repo) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/system/rollback", + strings.NewReader(`{"version":"0.1.146"}`)) + req.Header.Set("Content-Type", "application/json") + canceledCtx, cancel := context.WithCancel(context.Background()) + cancel() + req = req.WithContext(canceledCtx) + req.Header.Set("Idempotency-Key", "disconnected-rollback") + router.ServeHTTP(rec, req) + + require.Equal(t, 1, updateSvc.rollbackToCall) + require.NoError(t, updateSvc.rollbackToCtxErr, + "versioned rollback must not observe the canceled request context") + require.True(t, updateSvc.rollbackToHasDeadline, + "detached rollback context must still be bounded by a deadline") + requireSystemLockStatus(t, repo, service.IdempotencyStatusSucceeded) +} + func TestSystemHandlerRollbackWithoutBodyUsesLegacyBackup(t *testing.T) { updateSvc := &systemHandlerUpdateServiceStub{} repo := newMemoryIdempotencyRepoStub() diff --git a/frontend/src/api/admin/system.ts b/frontend/src/api/admin/system.ts index eb3db9b06c..1c057c8494 100644 --- a/frontend/src/api/admin/system.ts +++ b/frontend/src/api/admin/system.ts @@ -61,12 +61,22 @@ export async function getRollbackVersions(): Promise<{ versions: RollbackVersion return data } +/** + * In-place update/rollback downloads a full release binary from GitHub, which + * can take several minutes on slow links. The global 30s axios timeout would + * abort the request mid-download (#4504), so these calls wait as long as the + * backend allows (15 minutes server-side). + */ +const UPDATE_REQUEST_TIMEOUT_MS = 15 * 60 * 1000 + /** * Perform system update * Downloads and applies the latest version */ export async function performUpdate(): Promise { - const { data } = await apiClient.post('/admin/system/update') + const { data } = await apiClient.post('/admin/system/update', undefined, { + timeout: UPDATE_REQUEST_TIMEOUT_MS + }) return data } @@ -77,7 +87,8 @@ export async function performUpdate(): Promise { export async function rollback(version?: string): Promise { const { data } = await apiClient.post( '/admin/system/rollback', - version ? { version } : undefined + version ? { version } : undefined, + { timeout: UPDATE_REQUEST_TIMEOUT_MS } ) return data } From 4a97491be6be5257234748411e54783ca400c6cd Mon Sep 17 00:00:00 2001 From: superman2003 <2112076433zcr@gmail.com> Date: Sun, 19 Jul 2026 22:40:58 +0800 Subject: [PATCH 40/59] fix(grok): let manual connection tests bypass the scheduling gate Grok OAuth 'test connection' reused the production GrokTokenProvider request path, whose eligibility gate rejects any account that is not currently schedulable (manual switch off, rate limited, overloaded, temporarily cooled down). Every such test failed before reaching xAI with the generic 'oauth refresh account state changed', while Codex/OpenAI account tests read credentials regardless of scheduling state (#4598). Add GetAccessTokenForManualTest and use it from the account test service: - skips the request-path scheduling eligibility gate so admins can probe excluded accounts; credential integrity checks (configured proxy present, refresh token present) still apply - refreshes expired tokens through the shared OAuthRefreshAPI without the request-path marker, so the lock protocol and DB re-read keep working while scheduling state no longer blocks the refresh; credential rotation for non-active accounts stays blocked inside RefreshIfNeeded - falls back to the still-valid current token when a refresh attempt fails or another worker holds the refresh lock, and otherwise surfaces the concrete refresh error instead of the generic message Fixes #4598 --- .../internal/service/account_test_service.go | 4 +- .../internal/service/grok_token_provider.go | 71 +++++++++ .../service/grok_token_provider_test.go | 144 ++++++++++++++++++ 3 files changed, 218 insertions(+), 1 deletion(-) diff --git a/backend/internal/service/account_test_service.go b/backend/internal/service/account_test_service.go index 11c3853221..9cf120b13d 100644 --- a/backend/internal/service/account_test_service.go +++ b/backend/internal/service/account_test_service.go @@ -711,7 +711,9 @@ func (s *AccountTestService) testGrokAccountConnection(c *gin.Context, account * return s.sendErrorAndEnd(c, "Grok token provider not configured") } var err error - authToken, err = s.grokTokenProvider.GetAccessToken(ctx, account) + // 手动测试不走生产调度资格门：关闭调度、限流/过载/临时冷却中的账号 + // 也应能被管理员探测（#4598），与 Codex/OpenAI 测试行为一致。 + authToken, err = s.grokTokenProvider.GetAccessTokenForManualTest(ctx, account) if err != nil { return s.sendErrorAndEnd(c, fmt.Sprintf("Failed to get Grok access token: %s", err.Error())) } diff --git a/backend/internal/service/grok_token_provider.go b/backend/internal/service/grok_token_provider.go index ae57cdc59e..984f56cc2c 100644 --- a/backend/internal/service/grok_token_provider.go +++ b/backend/internal/service/grok_token_provider.go @@ -167,6 +167,77 @@ func (p *GrokTokenProvider) GetAccessToken(ctx context.Context, account *Account return accessToken, nil } +// GetAccessTokenForManualTest returns an access token for an admin-initiated +// "test connection" probe. Unlike GetAccessToken it does not apply the +// request-path scheduling eligibility gate (manual Schedulable switch, +// rate-limit / overload / temp-unschedulable cooldowns): a manual test exists +// precisely to check accounts in those states, matching how Codex/OpenAI +// account tests read credentials regardless of scheduling state (#4598). +// +// Credential integrity still applies: the configured-proxy-missing check, the +// shared refresh lock protocol, and the refresh API's own account re-read. +// Credential rotation for non-active (disabled/error) accounts remains +// blocked inside RefreshIfNeeded; their still-valid tokens are probed as-is. +func (p *GrokTokenProvider) GetAccessTokenForManualTest(ctx context.Context, account *Account) (string, error) { + if account == nil { + return "", errors.New("account is nil") + } + if account.Platform != PlatformGrok || account.Type != AccountTypeOAuth { + return "", errors.New("not a grok oauth account") + } + if account.ProxyID != nil && account.Proxy == nil { + return "", errGrokOAuthConfiguredProxyMiss + } + if strings.TrimSpace(account.GetGrokRefreshToken()) == "" { + return "", errGrokOAuthRefreshTokenMissing + } + + accessToken := strings.TrimSpace(account.GetGrokAccessToken()) + expiresAt := account.GetCredentialAsTime("expires_at") + tokenValid := accessToken != "" && expiresAt != nil && time.Now().Before(*expiresAt) + if accessToken != "" && expiresAt != nil && time.Until(*expiresAt) > grokTokenRefreshSkew { + return accessToken, nil + } + + if p.refreshAPI == nil || p.executor == nil { + if tokenValid { + return accessToken, nil + } + return "", errGrokOAuthRefreshNotConfigured + } + + // Deliberately not marked as a request-path refresh: the request path + // re-applies scheduling eligibility inside RefreshIfNeeded, which is + // exactly what a manual test must bypass. + refreshCtx, cancel := context.WithTimeout(ctx, grokRequestRefreshTimeout) + defer cancel() + result, err := p.refreshAPI.RefreshIfNeeded(refreshCtx, account, p.executor, grokTokenRefreshSkew) + if err != nil { + if tokenValid { + return accessToken, nil + } + return "", err + } + if result != nil && result.LockHeld { + if tokenValid { + return accessToken, nil + } + return "", errors.New("token refresh is already in progress on another worker; retry in a few seconds") + } + if result != nil && result.Account != nil { + account = result.Account + } + + accessToken = strings.TrimSpace(account.GetGrokAccessToken()) + if accessToken == "" { + return "", errGrokOAuthAccessTokenMissing + } + if latestExpiry := account.GetCredentialAsTime("expires_at"); latestExpiry != nil && !time.Now().Before(*latestExpiry) { + return "", errGrokOAuthAccessTokenExpired + } + return accessToken, nil +} + func (p *GrokTokenProvider) waitForRefreshedToken(ctx context.Context, account *Account, cacheKey string) (string, error) { waitCtx, cancel := context.WithTimeout(ctx, grokRefreshLockWaitTimeout) defer cancel() diff --git a/backend/internal/service/grok_token_provider_test.go b/backend/internal/service/grok_token_provider_test.go index 7c68e55455..fbfbc68861 100644 --- a/backend/internal/service/grok_token_provider_test.go +++ b/backend/internal/service/grok_token_provider_test.go @@ -287,6 +287,150 @@ func TestGrokTokenProviderRejectsStaleDBTokenWithoutExpiry(t *testing.T) { require.Empty(t, token) } +// TestGrokTokenProviderManualTestBypassesSchedulingGate reproduces #4598: +// admins must be able to run "test connection" against accounts that the +// scheduler currently excludes (manual switch off, rate limited, overloaded, +// temporarily cooled down). The production request path keeps rejecting them. +func TestGrokTokenProviderManualTestBypassesSchedulingGate(t *testing.T) { + future := time.Now().Add(time.Hour) + tests := []struct { + name string + mutate func(*Account) + }{ + {name: "not schedulable", mutate: func(account *Account) { account.Schedulable = false }}, + {name: "temporarily unschedulable", mutate: func(account *Account) { account.TempUnschedulableUntil = &future }}, + {name: "rate limited", mutate: func(account *Account) { account.RateLimitResetAt = &future }}, + {name: "overloaded", mutate: func(account *Account) { account.OverloadUntil = &future }}, + {name: "disabled by error", mutate: func(account *Account) { account.Status = StatusError }}, + } + + for index, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + account := expiredGrokOAuthAccountForCredentialTest(int64(120 + index)) + account.Credentials["access_token"] = "still-valid-token" + account.Credentials["expires_at"] = time.Now().Add(2 * grokTokenRefreshSkew).UTC().Format(time.RFC3339) + tt.mutate(account) + provider := NewGrokTokenProvider(&tokenRefreshAccountRepo{}, &grokTokenCacheForProviderTest{}) + + // Production request path keeps excluding this account. + _, requestErr := provider.GetAccessToken(context.Background(), account) + require.ErrorIs(t, requestErr, errOAuthRefreshAccountStateChanged) + + // Manual test path returns the valid credential for probing. + token, err := provider.GetAccessTokenForManualTest(context.Background(), account) + require.NoError(t, err) + require.Equal(t, "still-valid-token", token) + }) + } +} + +func TestGrokTokenProviderManualTestRefreshesExpiredTokenWhileUnschedulable(t *testing.T) { + t.Setenv(xai.EnvBaseURL, xai.DefaultCLIBaseURL) + + expiredAt := time.Now().Add(-time.Minute).UTC().Format(time.RFC3339) + account := &Account{ + ID: 130, + Platform: PlatformGrok, + Type: AccountTypeOAuth, + Status: StatusActive, + Schedulable: false, + Credentials: map[string]any{ + "access_token": "expired-access-token", + "refresh_token": "refresh-token", + "expires_at": expiredAt, + "base_url": xai.DefaultCLIBaseURL, + "client_id": "client-id", + }, + } + repo := &tokenRefreshAccountRepo{} + repo.accountsByID = map[int64]*Account{130: account} + cache := &grokTokenCacheForProviderTest{lockResult: true} + oauthSvc := NewGrokOAuthService(nil, &grokOAuthClientStub{ + refreshResponse: &xai.TokenResponse{ + AccessToken: "manual-test-refreshed-token", + TokenType: "Bearer", + ExpiresIn: 3600, + }, + }) + defer oauthSvc.Stop() + + provider := NewGrokTokenProvider(repo, cache) + provider.SetRefreshAPI(NewOAuthRefreshAPI(repo, cache), NewGrokTokenRefresher(oauthSvc)) + + token, err := provider.GetAccessTokenForManualTest(context.Background(), account) + require.NoError(t, err) + require.Equal(t, "manual-test-refreshed-token", token) + require.Equal(t, 1, repo.updateCredentialsCalls) +} + +func TestGrokTokenProviderManualTestFallsBackToValidTokenOnRefreshFailure(t *testing.T) { + account := &Account{ + ID: 131, + Platform: PlatformGrok, + Type: AccountTypeOAuth, + Status: StatusActive, + Schedulable: false, + Credentials: map[string]any{ + "access_token": "near-expiry-token", + "refresh_token": "refresh-token", + // Inside the refresh window but not expired yet. + "expires_at": time.Now().Add(10 * time.Minute).UTC().Format(time.RFC3339), + }, + } + repo := &tokenRefreshAccountRepo{} + repo.accountsByID = map[int64]*Account{131: account} + cache := &grokTokenCacheForProviderTest{lockResult: true} + provider := NewGrokTokenProvider(repo, cache) + provider.SetRefreshAPI(NewOAuthRefreshAPI(repo, cache), &tokenRefresherStub{ + err: errors.New("upstream refresh unavailable"), + }) + + token, err := provider.GetAccessTokenForManualTest(context.Background(), account) + require.NoError(t, err) + require.Equal(t, "near-expiry-token", token) +} + +func TestGrokTokenProviderManualTestReportsRefreshFailureWhenTokenExpired(t *testing.T) { + account := expiredGrokOAuthAccountForCredentialTest(132) + account.Schedulable = false + repo := &tokenRefreshAccountRepo{} + repo.accountsByID = map[int64]*Account{account.ID: account} + cache := &grokTokenCacheForProviderTest{lockResult: true} + provider := NewGrokTokenProvider(repo, cache) + provider.SetRefreshAPI(NewOAuthRefreshAPI(repo, cache), &tokenRefresherStub{ + err: errors.New("invalid_client: client credentials rejected"), + }) + + token, err := provider.GetAccessTokenForManualTest(context.Background(), account) + require.Error(t, err) + require.Empty(t, token) + require.Contains(t, err.Error(), "invalid_client") +} + +func TestGrokTokenProviderManualTestLockHeldWithExpiredTokenReturnsSpecificError(t *testing.T) { + account := expiredGrokOAuthAccountForCredentialTest(133) + repo := &tokenRefreshAccountRepo{} + repo.accountsByID = map[int64]*Account{account.ID: account} + cache := &grokTokenCacheForProviderTest{lockResult: false} + provider := NewGrokTokenProvider(repo, cache) + provider.SetRefreshAPI(NewOAuthRefreshAPI(repo, cache), &tokenRefresherStub{}) + + token, err := provider.GetAccessTokenForManualTest(context.Background(), account) + require.Error(t, err) + require.Empty(t, token) + require.Contains(t, err.Error(), "refresh is already in progress") +} + +func TestGrokTokenProviderManualTestRequiresRefreshToken(t *testing.T) { + account := expiredGrokOAuthAccountForCredentialTest(134) + delete(account.Credentials, "refresh_token") + provider := NewGrokTokenProvider(&tokenRefreshAccountRepo{}, &grokTokenCacheForProviderTest{}) + + token, err := provider.GetAccessTokenForManualTest(context.Background(), account) + require.ErrorIs(t, err, errGrokOAuthRefreshTokenMissing) + require.Empty(t, token) +} + func TestGrokTokenProviderRejectsIneligibleSelectedAccountBeforeWarmCache(t *testing.T) { future := time.Now().Add(time.Hour) tests := []struct { From 33796030222a16ec40a5291e0c29deafe5e2babd Mon Sep 17 00:00:00 2001 From: cugxuan Date: Sun, 19 Jul 2026 22:43:35 +0800 Subject: [PATCH 41/59] feat(subscription): show expiry times to minute --- .../__tests__/formatDateTimeToMinute.spec.ts | 15 +++++++++++++ frontend/src/utils/format.ts | 21 +++++++++++++++++++ .../src/views/admin/SubscriptionsView.vue | 6 +++--- frontend/src/views/user/SubscriptionsView.vue | 4 ++-- 4 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 frontend/src/utils/__tests__/formatDateTimeToMinute.spec.ts diff --git a/frontend/src/utils/__tests__/formatDateTimeToMinute.spec.ts b/frontend/src/utils/__tests__/formatDateTimeToMinute.spec.ts new file mode 100644 index 0000000000..1e881adb18 --- /dev/null +++ b/frontend/src/utils/__tests__/formatDateTimeToMinute.spec.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from 'vitest' + +import { formatDateTimeToMinute } from '../format' + +describe('formatDateTimeToMinute', () => { + it('formats local date and time without seconds', () => { + const value = new Date(2026, 6, 19, 20, 30, 45) + + expect(formatDateTimeToMinute(value, 'en-GB')).toBe('19/07/2026, 20:30') + }) + + it('returns an empty string for an invalid date', () => { + expect(formatDateTimeToMinute(new Date('invalid'), 'en-GB')).toBe('') + }) +}) diff --git a/frontend/src/utils/format.ts b/frontend/src/utils/format.ts index 481fe397ef..787bd4b545 100644 --- a/frontend/src/utils/format.ts +++ b/frontend/src/utils/format.ts @@ -149,6 +149,27 @@ export function formatDateTime( return formatDate(date, options, localeOverride) } +/** + * 格式化日期时间（精确到分钟） + */ +export function formatDateTimeToMinute( + date: string | Date | null | undefined, + localeOverride?: string +): string { + return formatDate( + date, + { + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + hour12: false + }, + localeOverride + ) +} + /** * 格式化为 date 控件值（YYYY-MM-DD，使用本地时间） */ diff --git a/frontend/src/views/admin/SubscriptionsView.vue b/frontend/src/views/admin/SubscriptionsView.vue index 3f6e73a8cd..85ae5f0a80 100644 --- a/frontend/src/views/admin/SubscriptionsView.vue +++ b/frontend/src/views/admin/SubscriptionsView.vue @@ -351,7 +351,7 @@ : 'text-gray-700 dark:text-gray-300' " > - {{ formatDateOnly(value) }} + {{ formatDateTimeToMinute(value) }}
{{ getDaysRemaining(value) }} {{ t('admin.subscriptions.daysRemaining') }} @@ -598,7 +598,7 @@ {{ extendingSubscription.expires_at - ? formatDateOnly(extendingSubscription.expires_at) + ? formatDateTimeToMinute(extendingSubscription.expires_at) : t('admin.subscriptions.noExpiration') }} @@ -764,7 +764,7 @@ import { adminAPI } from '@/api/admin' import type { UserSubscription, Group, GroupPlatform, SubscriptionType } from '@/types' import type { SimpleUser } from '@/api/admin/usage' import type { Column } from '@/components/common/types' -import { formatDateOnly } from '@/utils/format' +import { formatDateTimeToMinute } from '@/utils/format' import { getPersistedPageSize } from '@/composables/usePersistedPageSize' import AppLayout from '@/components/layout/AppLayout.vue' import TablePageLayout from '@/components/layout/TablePageLayout.vue' diff --git a/frontend/src/views/user/SubscriptionsView.vue b/frontend/src/views/user/SubscriptionsView.vue index b76f87582b..744529848a 100644 --- a/frontend/src/views/user/SubscriptionsView.vue +++ b/frontend/src/views/user/SubscriptionsView.vue @@ -256,7 +256,7 @@ import subscriptionsAPI from '@/api/subscriptions' import type { UserSubscription } from '@/types' import AppLayout from '@/components/layout/AppLayout.vue' import Icon from '@/components/icons/Icon.vue' -import { formatDateOnly } from '@/utils/format' +import { formatDateTimeToMinute } from '@/utils/format' import { hasPeakRate, formatPeakRateWindow, serverTimezoneLabel } from '@/utils/peak-rate' import { platformBorderClass, platformBadgeClass, platformButtonClass, platformLabel } from '@/utils/platformColors' import { getRemainingDurationParts, isOneTimeDailyQuota, type RemainingDurationParts } from '@/utils/subscriptionQuota' @@ -322,7 +322,7 @@ function formatExpirationDate(expiresAt: string): string { return t('userSubscriptions.status.expired') } - const dateStr = formatDateOnly(expires) + const dateStr = formatDateTimeToMinute(expires) if (days === 0) { return `${dateStr} (${t('common.today')})` From 041db5d8245cb898b99142b1af737c77a3eae679 Mon Sep 17 00:00:00 2001 From: Jlypx Date: Mon, 20 Jul 2026 00:08:26 +0800 Subject: [PATCH 42/59] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E8=87=AA?= =?UTF-8?q?=E5=AE=9A=E4=B9=89=E5=AE=A2=E6=88=B7=E7=AB=AF=20IP=20=E8=AF=B7?= =?UTF-8?q?=E6=B1=82=E5=A4=B4=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- backend/internal/config/config.go | 102 +++++++++++++++++---- backend/internal/config/config_test.go | 122 +++++++++++++++++++++++++ 2 files changed, 208 insertions(+), 16 deletions(-) diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 6aef080721..c60ef2a461 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -7,6 +7,7 @@ import ( "fmt" "log/slog" "math" + "net/textproto" "net/url" "os" "strings" @@ -14,6 +15,7 @@ import ( "time" "github.com/spf13/viper" + "golang.org/x/net/http/httpguts" ) const ( @@ -668,6 +670,13 @@ type CORSConfig struct { AllowCredentials bool `mapstructure:"allow_credentials"` } +const MaxForwardedClientIPHeaders = 16 + +type ForwardedClientIPSettings struct { + TrustForwardedIP bool + Headers []string +} + type SecurityConfig struct { URLAllowlist URLAllowlistConfig `mapstructure:"url_allowlist"` ResponseHeaders ResponseHeaderConfig `mapstructure:"response_headers"` @@ -676,19 +685,58 @@ type SecurityConfig struct { ProxyProbe ProxyProbeConfig `mapstructure:"proxy_probe"` // TrustForwardedIPForAPIKeyACL enables legacy raw forwarded-header takeover. // When disabled, server.trusted_proxies is authoritative for all client-IP consumers. - TrustForwardedIPForAPIKeyACL bool `mapstructure:"trust_forwarded_ip_for_api_key_acl"` - trustForwardedIPForAPIKeyACLLive *atomic.Bool `mapstructure:"-"` + TrustForwardedIPForAPIKeyACL bool `mapstructure:"trust_forwarded_ip_for_api_key_acl"` + ForwardedClientIPHeaders []string `mapstructure:"forwarded_client_ip_headers" json:"forwarded_client_ip_headers" yaml:"forwarded_client_ip_headers"` + forwardedClientIPSettingsLive atomic.Pointer[ForwardedClientIPSettings] `mapstructure:"-" json:"-" yaml:"-"` +} + +func NormalizeForwardedClientIPHeaders(headers []string) ([]string, error) { + normalized := make([]string, 0, len(headers)) + seen := make(map[string]struct{}, len(headers)) + for _, header := range headers { + header = strings.TrimSpace(header) + if !httpguts.ValidHeaderFieldName(header) { + return nil, fmt.Errorf("invalid HTTP header field name %q", header) + } + canonical := textproto.CanonicalMIMEHeaderKey(header) + key := strings.ToLower(canonical) + if _, exists := seen[key]; exists { + continue + } + if len(normalized) == MaxForwardedClientIPHeaders { + return nil, fmt.Errorf("forwarded client IP headers must contain at most %d unique names", MaxForwardedClientIPHeaders) + } + seen[key] = struct{}{} + normalized = append(normalized, canonical) + } + return normalized, nil +} + +func cloneForwardedClientIPHeaders(headers []string) []string { + if len(headers) == 0 { + return []string{} + } + return append([]string(nil), headers...) +} + +func (c *Config) ForwardedClientIPSettings() ForwardedClientIPSettings { + if c == nil { + return ForwardedClientIPSettings{Headers: []string{}} + } + if snapshot := c.Security.forwardedClientIPSettingsLive.Load(); snapshot != nil { + return ForwardedClientIPSettings{ + TrustForwardedIP: snapshot.TrustForwardedIP, + Headers: cloneForwardedClientIPHeaders(snapshot.Headers), + } + } + return ForwardedClientIPSettings{ + TrustForwardedIP: c.Security.TrustForwardedIPForAPIKeyACL, + Headers: cloneForwardedClientIPHeaders(c.Security.ForwardedClientIPHeaders), + } } func (c *Config) TrustForwardedIPForAPIKeyACL() bool { - if c == nil { - return false - } - live := c.Security.trustForwardedIPForAPIKeyACLLive - if live == nil { - return c.Security.TrustForwardedIPForAPIKeyACL - } - return live.Load() + return c.ForwardedClientIPSettings().TrustForwardedIP } // ForwardedClientIPTrustEnabled reports whether the legacy forwarded-header @@ -697,15 +745,22 @@ func (c *Config) ForwardedClientIPTrustEnabled() bool { return c != nil && c.TrustForwardedIPForAPIKeyACL() } +func (c *Config) SetForwardedClientIPSettings(enabled bool, headers []string) { + if c == nil { + return + } + headers = cloneForwardedClientIPHeaders(headers) + c.Security.forwardedClientIPSettingsLive.Store(&ForwardedClientIPSettings{ + TrustForwardedIP: enabled, + Headers: headers, + }) +} + func (c *Config) SetTrustForwardedIPForAPIKeyACL(enabled bool) { if c == nil { return } - c.Security.TrustForwardedIPForAPIKeyACL = enabled - if c.Security.trustForwardedIPForAPIKeyACLLive == nil { - c.Security.trustForwardedIPForAPIKeyACLLive = &atomic.Bool{} - } - c.Security.trustForwardedIPForAPIKeyACLLive.Store(enabled) + c.SetForwardedClientIPSettings(enabled, c.ForwardedClientIPSettings().Headers) } type URLAllowlistConfig struct { @@ -1574,6 +1629,7 @@ func load(allowMissingJWTSecret bool) (*Config, error) { // 配置文件不存在时使用默认值 } trustedProxiesEnv, trustedProxiesEnvConfigured := os.LookupEnv("SERVER_TRUSTED_PROXIES") + forwardedClientIPHeadersEnv, forwardedClientIPHeadersEnvConfigured := os.LookupEnv("SECURITY_FORWARDED_CLIENT_IP_HEADERS") trustedProxiesConfigured := viper.InConfig("server.trusted_proxies") || viper.IsSet("server.trusted_proxies") || trustedProxiesEnvConfigured @@ -1584,6 +1640,9 @@ func load(allowMissingJWTSecret bool) (*Config, error) { if trustedProxiesEnvConfigured { cfg.Server.TrustedProxies = normalizeStringSlice(strings.Split(trustedProxiesEnv, ",")) } + if forwardedClientIPHeadersEnvConfigured { + cfg.Security.ForwardedClientIPHeaders = normalizeStringSlice(strings.Split(forwardedClientIPHeadersEnv, ",")) + } cfg.Server.TrustedProxiesConfigured = trustedProxiesConfigured if cfg.Gateway.OpenAIScheduler.StickyEscapeTTFTMs == 0 { cfg.Gateway.OpenAIScheduler.StickyEscapeTTFTMs = 15000 @@ -1640,7 +1699,12 @@ func load(allowMissingJWTSecret bool) (*Config, error) { cfg.Security.ResponseHeaders.AdditionalAllowed = normalizeStringSlice(cfg.Security.ResponseHeaders.AdditionalAllowed) cfg.Security.ResponseHeaders.ForceRemove = normalizeStringSlice(cfg.Security.ResponseHeaders.ForceRemove) cfg.Security.CSP.Policy = strings.TrimSpace(cfg.Security.CSP.Policy) - cfg.SetTrustForwardedIPForAPIKeyACL(cfg.Security.TrustForwardedIPForAPIKeyACL) + forwardedClientIPHeaders, err := NormalizeForwardedClientIPHeaders(cfg.Security.ForwardedClientIPHeaders) + if err != nil { + return nil, fmt.Errorf("security.forwarded_client_ip_headers: %w", err) + } + cfg.Security.ForwardedClientIPHeaders = forwardedClientIPHeaders + cfg.SetForwardedClientIPSettings(cfg.Security.TrustForwardedIPForAPIKeyACL, forwardedClientIPHeaders) cfg.Log.Level = strings.ToLower(strings.TrimSpace(cfg.Log.Level)) cfg.Log.Format = strings.ToLower(strings.TrimSpace(cfg.Log.Format)) cfg.Log.ServiceName = strings.TrimSpace(cfg.Log.ServiceName) @@ -2244,6 +2308,12 @@ func setDefaults() { } func (c *Config) Validate() error { + forwardedClientIPHeaders, err := NormalizeForwardedClientIPHeaders(c.Security.ForwardedClientIPHeaders) + if err != nil { + return fmt.Errorf("security.forwarded_client_ip_headers: %w", err) + } + c.Security.ForwardedClientIPHeaders = forwardedClientIPHeaders + c.SetForwardedClientIPSettings(c.Security.TrustForwardedIPForAPIKeyACL, forwardedClientIPHeaders) if c.Server.ReadHeaderTimeout < 1 || c.Server.ReadHeaderTimeout > 60 { return fmt.Errorf("server.read_header_timeout must be between 1 and 60 seconds") } diff --git a/backend/internal/config/config_test.go b/backend/internal/config/config_test.go index 20da10a6a7..dece5fd5ae 100644 --- a/backend/internal/config/config_test.go +++ b/backend/internal/config/config_test.go @@ -1,10 +1,12 @@ package config import ( + "fmt" "math" "os" "path/filepath" "strings" + "sync" "testing" "time" @@ -50,6 +52,126 @@ func TestLoadHTTPIngressSafetyDefaults(t *testing.T) { require.Equal(t, 16384, cfg.APIKeyAuth.InvalidAbuse.Capacity) } +func TestNormalizeForwardedClientIPHeaders(t *testing.T) { + headers, err := NormalizeForwardedClientIPHeaders([]string{ + " x-cdn-client-ip ", + "X-CDN-CLIENT-IP", + "true-client-ip", + }) + require.NoError(t, err) + require.Equal(t, []string{"X-Cdn-Client-Ip", "True-Client-Ip"}, headers) + + _, err = NormalizeForwardedClientIPHeaders([]string{"X Invalid"}) + require.ErrorContains(t, err, "invalid HTTP header field name") +} + +func TestNormalizeForwardedClientIPHeadersLimit(t *testing.T) { + headers := make([]string, 0, MaxForwardedClientIPHeaders+1) + for i := 0; i <= MaxForwardedClientIPHeaders; i++ { + headers = append(headers, fmt.Sprintf("X-CDN-IP-%d", i)) + } + + _, err := NormalizeForwardedClientIPHeaders(headers) + require.ErrorContains(t, err, "at most 16 unique names") +} + +func TestLoadForwardedClientIPHeadersNormalizesAndSnapshots(t *testing.T) { + resetViperWithJWTSecret(t) + viper.Set("security.forwarded_client_ip_headers", []string{" x-cdn-ip ", "X-CDN-IP", "true-client-ip"}) + + cfg, err := Load() + require.NoError(t, err) + snapshot := cfg.ForwardedClientIPSettings() + require.Equal(t, []string{"X-Cdn-Ip", "True-Client-Ip"}, snapshot.Headers) + + snapshot.Headers[0] = "X-Mutated" + require.Equal(t, []string{"X-Cdn-Ip", "True-Client-Ip"}, cfg.ForwardedClientIPSettings().Headers) +} + +func TestForwardedClientIPSettingsConcurrentPublication(t *testing.T) { + cfg := &Config{} + cfg.SetForwardedClientIPSettings(true, []string{"X-Public-A"}) + + const iterations = 2000 + start := make(chan struct{}) + errCh := make(chan error, 8) + var wg sync.WaitGroup + + for _, settings := range []ForwardedClientIPSettings{ + {TrustForwardedIP: true, Headers: []string{"X-Public-A"}}, + {TrustForwardedIP: false, Headers: []string{"X-Public-B"}}, + } { + settings := settings + wg.Add(1) + go func() { + defer wg.Done() + <-start + for i := 0; i < iterations; i++ { + cfg.SetForwardedClientIPSettings(settings.TrustForwardedIP, settings.Headers) + } + }() + } + + for i := 0; i < cap(errCh); i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + for j := 0; j < iterations; j++ { + snapshot := cfg.ForwardedClientIPSettings() + validA := snapshot.TrustForwardedIP && len(snapshot.Headers) == 1 && snapshot.Headers[0] == "X-Public-A" + validB := !snapshot.TrustForwardedIP && len(snapshot.Headers) == 1 && snapshot.Headers[0] == "X-Public-B" + if !validA && !validB { + errCh <- fmt.Errorf("observed inconsistent forwarded IP settings: %+v", snapshot) + return + } + } + }() + } + + close(start) + wg.Wait() + close(errCh) + for err := range errCh { + require.NoError(t, err) + } +} + +func TestLoadForwardedClientIPHeadersFromEnvironment(t *testing.T) { + resetViperWithJWTSecret(t) + t.Setenv("SECURITY_FORWARDED_CLIENT_IP_HEADERS", " x-cdn-ip , X-CDN-IP, true-client-ip ") + + cfg, err := Load() + require.NoError(t, err) + require.Equal(t, []string{"X-Cdn-Ip", "True-Client-Ip"}, cfg.ForwardedClientIPSettings().Headers) +} + +func TestLoadExplicitEmptyForwardedClientIPHeadersFromEnvironment(t *testing.T) { + resetViperWithJWTSecret(t) + viper.Set("security.forwarded_client_ip_headers", []string{"X-Yaml-IP"}) + t.Setenv("SECURITY_FORWARDED_CLIENT_IP_HEADERS", "") + + cfg, err := Load() + require.NoError(t, err) + require.Empty(t, cfg.ForwardedClientIPSettings().Headers) +} + +func TestLoadRejectsInvalidForwardedClientIPHeaderFromEnvironment(t *testing.T) { + resetViperWithJWTSecret(t) + t.Setenv("SECURITY_FORWARDED_CLIENT_IP_HEADERS", "X-Valid-IP, X Invalid") + + _, err := Load() + require.ErrorContains(t, err, "security.forwarded_client_ip_headers") +} + +func TestLoadRejectsInvalidForwardedClientIPHeader(t *testing.T) { + resetViperWithJWTSecret(t) + viper.Set("security.forwarded_client_ip_headers", []string{"X Invalid"}) + + _, err := Load() + require.ErrorContains(t, err, "security.forwarded_client_ip_headers") +} + func TestLoadExplicitEmptyTrustedProxiesEnablesConfiguredMode(t *testing.T) { resetViperWithJWTSecret(t) viper.Set("server.trusted_proxies", []string{}) From 496005d68812e442cf641b7728fff58133d12607 Mon Sep 17 00:00:00 2001 From: Jlypx Date: Mon, 20 Jul 2026 00:08:36 +0800 Subject: [PATCH 43/59] =?UTF-8?q?fix:=20=E6=8C=89=E8=87=AA=E5=AE=9A?= =?UTF-8?q?=E4=B9=89=E8=AF=B7=E6=B1=82=E5=A4=B4=E8=A7=A3=E6=9E=90=E5=AE=A2?= =?UTF-8?q?=E6=88=B7=E7=AB=AF=20IP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- backend/internal/pkg/ip/ip.go | 104 ++++++++++++++++++++----- backend/internal/pkg/ip/ip_test.go | 117 +++++++++++++++++++++++++++++ 2 files changed, 201 insertions(+), 20 deletions(-) diff --git a/backend/internal/pkg/ip/ip.go b/backend/internal/pkg/ip/ip.go index 2e74665b0b..ff1347d809 100644 --- a/backend/internal/pkg/ip/ip.go +++ b/backend/internal/pkg/ip/ip.go @@ -8,22 +8,46 @@ import ( "github.com/gin-gonic/gin" ) -const legacyForwardedIPTrustKey = "sub2api.legacy_forwarded_ip_trust" +const forwardedIPSettingsKey = "sub2api.forwarded_ip_settings" + +type forwardedIPSettings struct { + trustForwarded bool + headers []string +} + +// SetForwardedIPSettings snapshots the forwarded-IP mode and custom header list +// for this request. +func SetForwardedIPSettings(c *gin.Context, enabled bool, headers []string) { + if c == nil { + return + } + c.Set(forwardedIPSettingsKey, forwardedIPSettings{ + trustForwarded: enabled, + headers: append([]string(nil), headers...), + }) +} // SetLegacyForwardedIPTrust records whether raw forwarding headers override // Gin's server.trusted_proxies chain for this request. func SetLegacyForwardedIPTrust(c *gin.Context, enabled bool) { - if c != nil { - c.Set(legacyForwardedIPTrustKey, enabled) + SetForwardedIPSettings(c, enabled, nil) +} + +func requestForwardedIPSettings(c *gin.Context) (forwardedIPSettings, bool) { + if c == nil { + return forwardedIPSettings{}, false } + value, ok := c.Get(forwardedIPSettingsKey) + if !ok { + return forwardedIPSettings{}, false + } + settings, ok := value.(forwardedIPSettings) + return settings, ok } func requestUsesLegacyForwardedIPTrust(c *gin.Context) bool { - if c == nil { - return true - } - enabled, ok := c.Get(legacyForwardedIPTrustKey) - return !ok || enabled == true + settings, ok := requestForwardedIPSettings(c) + return !ok || settings.trustForwarded } // GetClientIP resolves the client address using the legacy forwarding-header @@ -38,15 +62,61 @@ func GetClientIP(c *gin.Context) string { return GetTrustedClientIP(c) } + settings, _ := requestForwardedIPSettings(c) + customIP, customFallback := resolveCustomForwardedClientIP(c, settings.headers) + if customIP != "" { + return customIP + } + // Preserve the historical precedence used by existing reverse-proxy // deployments, while skipping an internal proxy address when a public XFF // value is available. This covers Docker/Nginx setups that accidentally // write the bridge address into X-Real-IP. + legacyIP, legacyFallback := resolveLegacyForwardedHeaderIP(c) + if legacyIP != "" { + return legacyIP + } + if customFallback != "" { + return customFallback + } + if legacyFallback != "" { + return legacyFallback + } + return normalizeIP(c.ClientIP()) +} + +func resolveCustomForwardedClientIP(c *gin.Context, headers []string) (string, string) { + if c == nil { + return "", "" + } + var fallback string + for _, header := range headers { + for _, value := range c.Request.Header.Values(header) { + for _, candidate := range strings.Split(value, ",") { + parsed := net.ParseIP(strings.TrimSpace(candidate)) + if parsed == nil { + continue + } + normalized := parsed.String() + if isPrivateIP(normalized) { + if fallback == "" { + fallback = normalized + } + continue + } + return normalized, fallback + } + } + } + return "", fallback +} + +func resolveLegacyForwardedHeaderIP(c *gin.Context) (string, string) { var fallback string if forwarded := normalizeIP(c.GetHeader("CF-Connecting-IP")); forwarded != "" { fallback = forwarded if !isPrivateIP(forwarded) { - return forwarded + return forwarded, fallback } } if realIP := normalizeIP(c.GetHeader("X-Real-IP")); realIP != "" { @@ -54,7 +124,7 @@ func GetClientIP(c *gin.Context) string { fallback = realIP } if !isPrivateIP(realIP) { - return realIP + return realIP, fallback } } if xff := c.GetHeader("X-Forwarded-For"); xff != "" { @@ -62,18 +132,14 @@ func GetClientIP(c *gin.Context) string { for _, candidate := range ips { candidate = strings.TrimSpace(candidate) if candidate != "" && !isPrivateIP(candidate) { - return normalizeIP(candidate) + return normalizeIP(candidate), fallback } } if fallback == "" && len(ips) > 0 { fallback = normalizeIP(strings.TrimSpace(ips[0])) } } - if fallback != "" { - return fallback - } - - return normalizeIP(c.ClientIP()) + return "", fallback } // GetTrustedClientIP 从 Gin 的可信代理解析链提取客户端 IP。 @@ -91,10 +157,8 @@ func GetTrustedClientIP(c *gin.Context) string { // client-IP resolution. When disabled, Gin's server.trusted_proxies chain is // authoritative. func GetSecurityClientIP(c *gin.Context, trustForwarded bool) string { - if c != nil { - if requestTrust, ok := c.Get(legacyForwardedIPTrustKey); ok { - trustForwarded = requestTrust == true - } + if requestSettings, ok := requestForwardedIPSettings(c); ok { + trustForwarded = requestSettings.trustForwarded } if trustForwarded { return GetClientIP(c) diff --git a/backend/internal/pkg/ip/ip_test.go b/backend/internal/pkg/ip/ip_test.go index d4347cab5a..505ade5eea 100644 --- a/backend/internal/pkg/ip/ip_test.go +++ b/backend/internal/pkg/ip/ip_test.go @@ -92,6 +92,101 @@ func TestGetSecurityClientIPSwitchEnabledUsesLegacyHeaders(t *testing.T) { require.Equal(t, "1.2.3.4", w.Body.String()) } +func TestGetSecurityClientIPCustomHeaderPrecedenceAndFallback(t *testing.T) { + gin.SetMode(gin.TestMode) + + tests := []struct { + name string + trustForward bool + headers []string + requestHeaders map[string]string + want string + }{ + { + name: "configured order precedes built-ins", + trustForward: true, + headers: []string{"X-CDN-First", "X-CDN-Second"}, + requestHeaders: map[string]string{ + "X-CDN-First": "198.51.100.10", + "X-CDN-Second": "203.0.113.20", + "CF-Connecting-IP": "8.8.8.8", + }, + want: "198.51.100.10", + }, + { + name: "comma candidates skip invalid and private values", + trustForward: true, + headers: []string{"X-CDN-First", "X-CDN-Second"}, + requestHeaders: map[string]string{ + "X-CDN-First": "not-an-ip, 10.0.0.8", + "X-CDN-Second": "also-bad, 203.0.113.9", + }, + want: "203.0.113.9", + }, + { + name: "legacy public header wins over custom private fallback", + trustForward: true, + headers: []string{"X-CDN-IP"}, + requestHeaders: map[string]string{ + "X-CDN-IP": "10.0.0.8", + "X-Real-IP": "1.2.3.4", + }, + want: "1.2.3.4", + }, + { + name: "custom private fallback retains configured precedence", + trustForward: true, + headers: []string{"X-CDN-IP"}, + requestHeaders: map[string]string{ + "X-CDN-IP": "10.0.0.8", + "X-Real-IP": "192.168.1.4", + }, + want: "10.0.0.8", + }, + { + name: "invalid custom value continues to built-ins", + trustForward: true, + headers: []string{"X-CDN-IP"}, + requestHeaders: map[string]string{ + "X-CDN-IP": "1.2.3.4:443", + "CF-Connecting-IP": "4.4.4.4", + }, + want: "4.4.4.4", + }, + { + name: "disabled mode ignores custom and legacy headers", + trustForward: false, + headers: []string{"X-CDN-IP"}, + requestHeaders: map[string]string{ + "X-CDN-IP": "1.2.3.4", + "X-Real-IP": "4.4.4.4", + }, + want: "9.9.9.9", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + r := gin.New() + require.NoError(t, r.SetTrustedProxies(nil)) + r.GET("/t", func(c *gin.Context) { + SetForwardedIPSettings(c, test.trustForward, test.headers) + c.String(200, GetSecurityClientIP(c, !test.trustForward)) + }) + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/t", nil) + req.RemoteAddr = "9.9.9.9:12345" + for name, value := range test.requestHeaders { + req.Header.Set(name, value) + } + r.ServeHTTP(w, req) + + require.Equal(t, test.want, w.Body.String()) + }) + } +} + func TestGetSecurityClientIPSwitchDisabledUsesConfiguredTrustedProxy(t *testing.T) { gin.SetMode(gin.TestMode) r := gin.New() @@ -125,6 +220,28 @@ func TestGetClientIPSwitchDisabledUsesTrustedProxyChain(t *testing.T) { require.Equal(t, "9.9.9.9", w.Body.String()) } +func TestGetSecurityClientIPRequestSnapshotCopiesCustomHeaders(t *testing.T) { + gin.SetMode(gin.TestMode) + + r := gin.New() + require.NoError(t, r.SetTrustedProxies(nil)) + r.GET("/t", func(c *gin.Context) { + headers := []string{"X-Original-IP"} + SetForwardedIPSettings(c, true, headers) + headers[0] = "X-Mutated-IP" + c.String(200, GetSecurityClientIP(c, false)) + }) + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/t", nil) + req.RemoteAddr = "9.9.9.9:12345" + req.Header.Set("X-Original-IP", "1.2.3.4") + req.Header.Set("X-Mutated-IP", "4.4.4.4") + r.ServeHTTP(w, req) + + require.Equal(t, "1.2.3.4", w.Body.String()) +} + func TestGetSecurityClientIPRequestSnapshotOverridesLiveFallback(t *testing.T) { gin.SetMode(gin.TestMode) From 9bc7d10c086e4b3bbba1e72ae092ef506c23d388 Mon Sep 17 00:00:00 2001 From: Jlypx Date: Mon, 20 Jul 2026 00:08:48 +0800 Subject: [PATCH 44/59] =?UTF-8?q?fix:=20=E5=BF=AB=E7=85=A7=E8=87=AA?= =?UTF-8?q?=E5=AE=9A=E4=B9=89=E5=AE=A2=E6=88=B7=E7=AB=AF=20IP=20=E8=AF=B7?= =?UTF-8?q?=E6=B1=82=E5=A4=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .../server/middleware/session_binding.go | 6 ++-- .../server/middleware/session_binding_test.go | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/backend/internal/server/middleware/session_binding.go b/backend/internal/server/middleware/session_binding.go index 3f4d04b935..19fca22032 100644 --- a/backend/internal/server/middleware/session_binding.go +++ b/backend/internal/server/middleware/session_binding.go @@ -17,12 +17,12 @@ import ( // 接管解析，关闭时使用 Gin 的 server.trusted_proxies 可信代理链。 func SessionBindingContext(cfg *config.Config) gin.HandlerFunc { return func(c *gin.Context) { - trustForwarded := cfg.TrustForwardedIPForAPIKeyACL() - ip.SetLegacyForwardedIPTrust(c, trustForwarded) + forwardedIPSettings := cfg.ForwardedClientIPSettings() + ip.SetForwardedIPSettings(c, forwardedIPSettings.TrustForwardedIP, forwardedIPSettings.Headers) userAgent := normalizePersistentText(c.Request.UserAgent(), maxPersistentUserAgentBytes) c.Request.Header.Set("User-Agent", userAgent) binding := &service.SessionBinding{ - IP: ip.GetSecurityClientIP(c, trustForwarded), + IP: ip.GetSecurityClientIP(c, forwardedIPSettings.TrustForwardedIP), UserAgent: userAgent, } c.Request = c.Request.WithContext(service.WithSessionBinding(c.Request.Context(), binding)) diff --git a/backend/internal/server/middleware/session_binding_test.go b/backend/internal/server/middleware/session_binding_test.go index 11a098d9fd..08e9448788 100644 --- a/backend/internal/server/middleware/session_binding_test.go +++ b/backend/internal/server/middleware/session_binding_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/pkg/ip" "github.com/Wei-Shaw/sub2api/internal/service" "github.com/gin-gonic/gin" @@ -55,6 +56,39 @@ func TestSessionBindingContextFollowsForwardedIPSwitch(t *testing.T) { } } +func TestSessionBindingContextSnapshotsForwardedModeAndHeaders(t *testing.T) { + gin.SetMode(gin.TestMode) + + cfg := &config.Config{} + cfg.SetForwardedClientIPSettings(true, []string{"X-Initial-IP"}) + + r := gin.New() + require.NoError(t, r.SetTrustedProxies(nil)) + r.Use(SessionBindingContext(cfg)) + r.GET("/t", func(c *gin.Context) { + binding := service.SessionBindingFromContext(c.Request.Context()) + require.NotNil(t, binding) + require.Equal(t, "1.2.3.4", binding.IP) + + cfg.SetForwardedClientIPSettings(false, []string{"X-Changed-IP"}) + require.Equal(t, "1.2.3.4", ip.GetSecurityClientIP(c, false)) + c.Status(200) + }) + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/t", nil) + req.RemoteAddr = "9.9.9.9:12345" + req.Header.Set("X-Initial-IP", "1.2.3.4") + req.Header.Set("X-Changed-IP", "4.4.4.4") + req.Header.Set("X-Real-IP", "8.8.8.8") + r.ServeHTTP(w, req) + + require.Equal(t, 200, w.Code) + runtimeSettings := cfg.ForwardedClientIPSettings() + require.False(t, runtimeSettings.TrustForwardedIP) + require.Equal(t, []string{"X-Changed-IP"}, runtimeSettings.Headers) +} + func TestSessionBindingContextBoundsPersistedUserAgent(t *testing.T) { cfg := &config.Config{} r := gin.New() From 8b8b6b31326213e37860e82e8590e644e8c03dc3 Mon Sep 17 00:00:00 2001 From: Jlypx Date: Mon, 20 Jul 2026 00:09:00 +0800 Subject: [PATCH 45/59] =?UTF-8?q?feat:=20=E5=AE=9A=E4=B9=89=E5=AE=A2?= =?UTF-8?q?=E6=88=B7=E7=AB=AF=20IP=20=E8=AF=B7=E6=B1=82=E5=A4=B4=E7=B3=BB?= =?UTF-8?q?=E7=BB=9F=E8=AE=BE=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- backend/internal/service/domain_constants.go | 1 + backend/internal/service/settings_view.go | 1 + 2 files changed, 2 insertions(+) diff --git a/backend/internal/service/domain_constants.go b/backend/internal/service/domain_constants.go index c99d751c87..e503cfacab 100644 --- a/backend/internal/service/domain_constants.go +++ b/backend/internal/service/domain_constants.go @@ -165,6 +165,7 @@ const ( // API Key IP 访问控制设置 SettingKeyAPIKeyACLTrustForwardedIP = "api_key_acl_trust_forwarded_ip" // API Key IP 白/黑名单是否信任转发 IP + SettingKeyForwardedClientIPHeaders = "forwarded_client_ip_headers" // 自定义 CDN 客户端 IP 请求头（JSON 数组） settingKeyForwardedClientIPModeV2 = "forwarded_client_ip_mode_v2_migrated" // TOTP 双因素认证设置 diff --git a/backend/internal/service/settings_view.go b/backend/internal/service/settings_view.go index 85ab619d14..c775e2ee77 100644 --- a/backend/internal/service/settings_view.go +++ b/backend/internal/service/settings_view.go @@ -42,6 +42,7 @@ type SystemSettings struct { TurnstileSecretKey string TurnstileSecretKeyConfigured bool APIKeyACLTrustForwardedIP bool + ForwardedClientIPHeaders []string // LinuxDo Connect OAuth 登录 LinuxDoConnectEnabled bool From f72958d53093390c1b3369294b835bed8667ea10 Mon Sep 17 00:00:00 2001 From: Jlypx Date: Mon, 20 Jul 2026 00:09:11 +0800 Subject: [PATCH 46/59] =?UTF-8?q?feat:=20=E6=8C=81=E4=B9=85=E5=8C=96?= =?UTF-8?q?=E5=AE=A2=E6=88=B7=E7=AB=AF=20IP=20=E8=AF=B7=E6=B1=82=E5=A4=B4?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- backend/internal/service/setting_parse.go | 43 ++++- backend/internal/service/setting_service.go | 56 ++++-- .../service/setting_service_update_test.go | 160 ++++++++++++++++-- backend/internal/service/setting_update.go | 13 +- backend/internal/service/wire.go | 4 +- 5 files changed, 242 insertions(+), 34 deletions(-) diff --git a/backend/internal/service/setting_parse.go b/backend/internal/service/setting_parse.go index 4481bf262d..adc1195c04 100644 --- a/backend/internal/service/setting_parse.go +++ b/backend/internal/service/setting_parse.go @@ -43,6 +43,14 @@ func (s *SettingService) InitializeDefaultSettings(ctx context.Context) error { if err != nil { return err } + forwardedClientIPHeaders := []string{} + if s != nil && s.cfg != nil { + forwardedClientIPHeaders = s.cfg.ForwardedClientIPSettings().Headers + } + forwardedClientIPHeadersJSON, err := json.Marshal(forwardedClientIPHeaders) + if err != nil { + return fmt.Errorf("marshal default forwarded client IP headers: %w", err) + } // 初始化默认设置 defaults := map[string]string{ @@ -55,6 +63,7 @@ func (s *SettingService) InitializeDefaultSettings(ctx context.Context) error { SettingKeyLoginAgreementUpdatedAt: defaultLoginAgreementDate, SettingKeyLoginAgreementDocuments: loginAgreementDocumentsJSON, SettingKeyAPIKeyACLTrustForwardedIP: "true", + SettingKeyForwardedClientIPHeaders: string(forwardedClientIPHeadersJSON), settingKeyForwardedClientIPModeV2: "true", SettingKeySiteName: "Sub2API", SettingKeySiteLogo: "", @@ -238,6 +247,21 @@ func (s *SettingService) InitializeDefaultSettings(ctx context.Context) error { return s.settingRepo.SetMultiple(ctx, defaults) } +func parseForwardedClientIPHeadersSetting(value string) ([]string, error) { + var headers []string + if err := json.Unmarshal([]byte(value), &headers); err != nil { + return nil, fmt.Errorf("parse forwarded_client_ip_headers: %w", err) + } + if headers == nil { + return nil, fmt.Errorf("parse forwarded_client_ip_headers: value must be a JSON array") + } + normalized, err := config.NormalizeForwardedClientIPHeaders(headers) + if err != nil { + return nil, fmt.Errorf("parse forwarded_client_ip_headers: %w", err) + } + return normalized, nil +} + // parseSettings 解析设置到结构体 func (s *SettingService) parseSettings(settings map[string]string) *SystemSettings { emailVerifyEnabled := settings[SettingKeyEmailVerifyEnabled] == "true" @@ -247,10 +271,24 @@ func (s *SettingService) parseSettings(settings map[string]string) *SystemSettin loginAgreementUpdatedAt = defaultLoginAgreementDate } apiKeyACLTrustForwardedIP := false + forwardedClientIPHeaders := []string{} + if s != nil && s.cfg != nil { + runtimeSettings := s.cfg.ForwardedClientIPSettings() + apiKeyACLTrustForwardedIP = runtimeSettings.TrustForwardedIP + forwardedClientIPHeaders = runtimeSettings.Headers + } if value, ok := settings[SettingKeyAPIKeyACLTrustForwardedIP]; ok { apiKeyACLTrustForwardedIP = value == "true" - } else if s != nil && s.cfg != nil { - apiKeyACLTrustForwardedIP = s.cfg.ForwardedClientIPTrustEnabled() + } + if value, ok := settings[SettingKeyForwardedClientIPHeaders]; ok { + parsed, err := parseForwardedClientIPHeadersSetting(value) + if err != nil { + slog.Error("invalid persisted forwarded client IP headers; forwarded trust disabled", "error", err) + apiKeyACLTrustForwardedIP = false + forwardedClientIPHeaders = []string{} + } else { + forwardedClientIPHeaders = parsed + } } result := &SystemSettings{ RegistrationEnabled: settings[SettingKeyRegistrationEnabled] == "true", @@ -278,6 +316,7 @@ func (s *SettingService) parseSettings(settings map[string]string) *SystemSettin TurnstileSiteKey: settings[SettingKeyTurnstileSiteKey], TurnstileSecretKeyConfigured: settings[SettingKeyTurnstileSecretKey] != "", APIKeyACLTrustForwardedIP: apiKeyACLTrustForwardedIP, + ForwardedClientIPHeaders: forwardedClientIPHeaders, SiteName: s.getStringOrDefault(settings, SettingKeySiteName, "Sub2API"), SiteLogo: settings[SettingKeySiteLogo], SiteSubtitle: s.getStringOrDefault(settings, SettingKeySiteSubtitle, "Subscription to API Conversion Platform"), diff --git a/backend/internal/service/setting_service.go b/backend/internal/service/setting_service.go index 608cc0320b..5f02110b46 100644 --- a/backend/internal/service/setting_service.go +++ b/backend/internal/service/setting_service.go @@ -2,6 +2,8 @@ package service import ( "context" + "encoding/json" + "errors" "fmt" "sync/atomic" @@ -215,42 +217,66 @@ func (s *SettingService) SetProxyRepository(repo ProxyRepository) { s.proxyRepo = repo } -func (s *SettingService) LoadAPIKeyACLTrustForwardedIPSetting(ctx context.Context) error { +func (s *SettingService) LoadForwardedClientIPSettings(ctx context.Context) error { if s == nil || s.cfg == nil || s.settingRepo == nil { return nil } values, err := s.settingRepo.GetMultiple(ctx, []string{ SettingKeyAPIKeyACLTrustForwardedIP, + SettingKeyForwardedClientIPHeaders, settingKeyForwardedClientIPModeV2, }) if err != nil { - s.cfg.SetTrustForwardedIPForAPIKeyACL(false) + s.cfg.SetForwardedClientIPSettings(false, nil) return fmt.Errorf("get forwarded client ip settings: %w", err) } enabled := s.cfg.Security.TrustForwardedIPForAPIKeyACL + headers := s.cfg.ForwardedClientIPSettings().Headers storedValue, hasStoredValue := values[SettingKeyAPIKeyACLTrustForwardedIP] if hasStoredValue { enabled = storedValue == "true" } - if values[settingKeyForwardedClientIPModeV2] != "true" { - updates := map[string]string{settingKeyForwardedClientIPModeV2: "true"} - // Before this migration, new installations persisted false by default. - // Restore compatibility only when no trusted-proxy policy was configured. - if hasStoredValue && !enabled && !s.cfg.Server.TrustedProxiesConfigured { - enabled = true - updates[SettingKeyAPIKeyACLTrustForwardedIP] = "true" - } - if err := s.settingRepo.SetMultiple(ctx, updates); err != nil { - s.cfg.SetTrustForwardedIPForAPIKeyACL(enabled) - return fmt.Errorf("migrate forwarded client ip setting: %w", err) + var headersErr error + if storedHeaders, ok := values[SettingKeyForwardedClientIPHeaders]; ok { + headers, headersErr = parseForwardedClientIPHeadersSetting(storedHeaders) + if headersErr != nil { + enabled = false + headers = []string{} + headersErr = fmt.Errorf("load forwarded client ip headers: %w", headersErr) } } - s.cfg.SetTrustForwardedIPForAPIKeyACL(enabled) - return nil + updates := make(map[string]string) + if _, hasStoredHeaders := values[SettingKeyForwardedClientIPHeaders]; !hasStoredHeaders { + headersJSON, marshalErr := json.Marshal(headers) + if marshalErr != nil { + headers = []string{} + headersErr = errors.Join(headersErr, fmt.Errorf("marshal forwarded client ip headers: %w", marshalErr)) + headersJSON = []byte("[]") + } + updates[SettingKeyForwardedClientIPHeaders] = string(headersJSON) + } + if values[settingKeyForwardedClientIPModeV2] != "true" { + updates[settingKeyForwardedClientIPModeV2] = "true" + // Before this migration, new installations persisted false by default. + // Restore compatibility only when no trusted-proxy policy was configured. + if headersErr == nil && hasStoredValue && !enabled && !s.cfg.Server.TrustedProxiesConfigured { + enabled = true + updates[SettingKeyAPIKeyACLTrustForwardedIP] = "true" + } + } + if len(updates) > 0 { + if err := s.settingRepo.SetMultiple(ctx, updates); err != nil { + s.cfg.SetForwardedClientIPSettings(enabled, headers) + return errors.Join(headersErr, fmt.Errorf("migrate forwarded client ip setting: %w", err)) + } + } + + s.cfg.SetForwardedClientIPSettings(enabled, headers) + return headersErr } // GetAllSettings 获取所有系统设置 diff --git a/backend/internal/service/setting_service_update_test.go b/backend/internal/service/setting_service_update_test.go index efbf646feb..3a4050f5b2 100644 --- a/backend/internal/service/setting_service_update_test.go +++ b/backend/internal/service/setting_service_update_test.go @@ -17,7 +17,8 @@ import ( ) type settingUpdateRepoStub struct { - updates map[string]string + updates map[string]string + setMultipleErr error } func (s *settingUpdateRepoStub) Get(ctx context.Context, key string) (*Setting, error) { @@ -41,7 +42,7 @@ func (s *settingUpdateRepoStub) SetMultiple(ctx context.Context, settings map[st for k, v := range settings { s.updates[k] = v } - return nil + return s.setMultipleErr } func (s *settingUpdateRepoStub) GetAll(ctx context.Context) (map[string]string, error) { @@ -99,8 +100,12 @@ func (s *forwardedIPMigrationRepoStub) Get(context.Context, string) (*Setting, e panic("unexpected Get call") } -func (s *forwardedIPMigrationRepoStub) GetValue(context.Context, string) (string, error) { - panic("unexpected GetValue call") +func (s *forwardedIPMigrationRepoStub) GetValue(_ context.Context, key string) (string, error) { + value, ok := s.values[key] + if !ok { + return "", ErrSettingNotFound + } + return value, nil } func (s *forwardedIPMigrationRepoStub) Set(context.Context, string, string) error { @@ -548,6 +553,16 @@ func TestSettingService_UpdateSettings_AntigravityUserAgentVersion(t *testing.T) require.Equal(t, "1.23.2", repo.updates[SettingKeyAntigravityUserAgentVersion]) } +func TestSettingService_InitializeDefaultSettingsPersistsConfiguredForwardedClientIPHeaders(t *testing.T) { + repo := &forwardedIPMigrationRepoStub{values: map[string]string{}} + cfg := &config.Config{} + cfg.SetForwardedClientIPSettings(true, []string{"X-Cdn-Ip", "True-Client-Ip"}) + svc := NewSettingService(repo, cfg) + + require.NoError(t, svc.InitializeDefaultSettings(context.Background())) + require.JSONEq(t, `["X-Cdn-Ip","True-Client-Ip"]`, repo.values[SettingKeyForwardedClientIPHeaders]) +} + func TestSettingService_UpdateSettings_APIKeyACLTrustForwardedIPRefreshesConfig(t *testing.T) { repo := &settingUpdateRepoStub{} cfg := &config.Config{} @@ -555,11 +570,51 @@ func TestSettingService_UpdateSettings_APIKeyACLTrustForwardedIPRefreshesConfig( err := svc.UpdateSettings(context.Background(), &SystemSettings{ APIKeyACLTrustForwardedIP: true, + ForwardedClientIPHeaders: []string{" x-cdn-ip ", "X-CDN-IP", "true-client-ip"}, }) require.NoError(t, err) require.Equal(t, "true", repo.updates[SettingKeyAPIKeyACLTrustForwardedIP]) - require.True(t, cfg.Security.TrustForwardedIPForAPIKeyACL) - require.True(t, cfg.TrustForwardedIPForAPIKeyACL()) + require.JSONEq(t, `["X-Cdn-Ip","True-Client-Ip"]`, repo.updates[SettingKeyForwardedClientIPHeaders]) + runtimeSettings := cfg.ForwardedClientIPSettings() + require.True(t, runtimeSettings.TrustForwardedIP) + require.Equal(t, []string{"X-Cdn-Ip", "True-Client-Ip"}, runtimeSettings.Headers) + + runtimeSettings.Headers[0] = "X-Mutated" + require.Equal(t, []string{"X-Cdn-Ip", "True-Client-Ip"}, cfg.ForwardedClientIPSettings().Headers) +} + +func TestSettingService_UpdateSettings_RejectsInvalidForwardedClientIPHeadersWithoutRefreshing(t *testing.T) { + repo := &settingUpdateRepoStub{} + cfg := &config.Config{} + cfg.SetForwardedClientIPSettings(true, []string{"X-Existing-IP"}) + svc := NewSettingService(repo, cfg) + + err := svc.UpdateSettings(context.Background(), &SystemSettings{ + ForwardedClientIPHeaders: []string{"X Invalid"}, + }) + + require.Error(t, err) + require.Nil(t, repo.updates) + runtimeSettings := cfg.ForwardedClientIPSettings() + require.True(t, runtimeSettings.TrustForwardedIP) + require.Equal(t, []string{"X-Existing-IP"}, runtimeSettings.Headers) +} + +func TestSettingService_UpdateSettings_WriteFailureDoesNotRefreshForwardedIPRuntime(t *testing.T) { + repo := &settingUpdateRepoStub{setMultipleErr: errors.New("database unavailable")} + cfg := &config.Config{} + cfg.SetForwardedClientIPSettings(false, []string{"X-Existing-IP"}) + svc := NewSettingService(repo, cfg) + + err := svc.UpdateSettings(context.Background(), &SystemSettings{ + APIKeyACLTrustForwardedIP: true, + ForwardedClientIPHeaders: []string{"X-New-IP"}, + }) + + require.ErrorContains(t, err, "database unavailable") + runtimeSettings := cfg.ForwardedClientIPSettings() + require.False(t, runtimeSettings.TrustForwardedIP) + require.Equal(t, []string{"X-Existing-IP"}, runtimeSettings.Headers) } func TestSettingService_ParseSettings_APIKeyACLTrustForwardedIPFallsBackToConfigWhenMissing(t *testing.T) { @@ -582,7 +637,34 @@ func TestSettingService_ParseSettings_APIKeyACLTrustForwardedIPUsesStoredValue(t require.False(t, got.APIKeyACLTrustForwardedIP) } -func TestSettingService_LoadAPIKeyACLTrustForwardedIPSettingMigration(t *testing.T) { +func TestSettingService_ParseSettings_ForwardedClientIPHeaders(t *testing.T) { + cfg := &config.Config{} + cfg.SetForwardedClientIPSettings(true, []string{"X-Config-IP"}) + svc := NewSettingService(&settingUpdateRepoStub{}, cfg) + + t.Run("stored value is normalized", func(t *testing.T) { + got := svc.parseSettings(map[string]string{ + SettingKeyForwardedClientIPHeaders: `[" x-cdn-ip ","X-CDN-IP","true-client-ip"]`, + }) + require.Equal(t, []string{"X-Cdn-Ip", "True-Client-Ip"}, got.ForwardedClientIPHeaders) + }) + + t.Run("missing value falls back to config", func(t *testing.T) { + got := svc.parseSettings(map[string]string{}) + require.Equal(t, []string{"X-Config-IP"}, got.ForwardedClientIPHeaders) + }) + + t.Run("malformed value disables forwarded trust", func(t *testing.T) { + got := svc.parseSettings(map[string]string{ + SettingKeyAPIKeyACLTrustForwardedIP: "true", + SettingKeyForwardedClientIPHeaders: `{"not":"an array"}`, + }) + require.False(t, got.APIKeyACLTrustForwardedIP) + require.Empty(t, got.ForwardedClientIPHeaders) + }) +} + +func TestSettingService_LoadForwardedClientIPSettingsMigration(t *testing.T) { tests := []struct { name string values map[string]string @@ -630,19 +712,67 @@ func TestSettingService_LoadAPIKeyACLTrustForwardedIPSettingMigration(t *testing cfg.Security.TrustForwardedIPForAPIKeyACL = test.configDefault svc := NewSettingService(repo, cfg) - require.NoError(t, svc.LoadAPIKeyACLTrustForwardedIPSetting(context.Background())) + require.NoError(t, svc.LoadForwardedClientIPSettings(context.Background())) require.Equal(t, test.wantEnabled, cfg.TrustForwardedIPForAPIKeyACL()) require.Equal(t, test.wantForwardedIPUpdate, repo.updates[SettingKeyAPIKeyACLTrustForwardedIP]) + require.JSONEq(t, `[]`, repo.updates[SettingKeyForwardedClientIPHeaders]) if test.wantMigrationMarkerSet { require.Equal(t, "true", repo.updates[settingKeyForwardedClientIPModeV2]) } else { - require.Nil(t, repo.updates) + require.NotContains(t, repo.updates, settingKeyForwardedClientIPModeV2) } }) } } -func TestSettingService_LoadAPIKeyACLTrustForwardedIPSettingReadFailureFailsClosed(t *testing.T) { +func TestSettingService_LoadForwardedClientIPSettingsLoadsHeaders(t *testing.T) { + repo := &forwardedIPMigrationRepoStub{values: map[string]string{ + SettingKeyAPIKeyACLTrustForwardedIP: "true", + SettingKeyForwardedClientIPHeaders: `[" x-cdn-ip ","true-client-ip"]`, + settingKeyForwardedClientIPModeV2: "true", + }} + cfg := &config.Config{} + svc := NewSettingService(repo, cfg) + + require.NoError(t, svc.LoadForwardedClientIPSettings(context.Background())) + runtimeSettings := cfg.ForwardedClientIPSettings() + require.True(t, runtimeSettings.TrustForwardedIP) + require.Equal(t, []string{"X-Cdn-Ip", "True-Client-Ip"}, runtimeSettings.Headers) + require.Nil(t, repo.updates) +} + +func TestSettingService_LoadForwardedClientIPSettingsMalformedHeadersDisablesCustomTrust(t *testing.T) { + repo := &forwardedIPMigrationRepoStub{values: map[string]string{ + SettingKeyAPIKeyACLTrustForwardedIP: "true", + SettingKeyForwardedClientIPHeaders: `["X Invalid"]`, + }} + cfg := &config.Config{} + svc := NewSettingService(repo, cfg) + + err := svc.LoadForwardedClientIPSettings(context.Background()) + + require.ErrorContains(t, err, "load forwarded client ip headers") + runtimeSettings := cfg.ForwardedClientIPSettings() + require.False(t, runtimeSettings.TrustForwardedIP) + require.Empty(t, runtimeSettings.Headers) + require.Equal(t, "true", repo.updates[settingKeyForwardedClientIPModeV2]) + require.NotContains(t, repo.updates, SettingKeyAPIKeyACLTrustForwardedIP) +} + +func TestSettingService_LoadForwardedClientIPSettingsBackfillsConfigHeaders(t *testing.T) { + repo := &forwardedIPMigrationRepoStub{values: map[string]string{ + settingKeyForwardedClientIPModeV2: "true", + }} + cfg := &config.Config{} + cfg.SetForwardedClientIPSettings(false, []string{"X-Config-IP"}) + svc := NewSettingService(repo, cfg) + + require.NoError(t, svc.LoadForwardedClientIPSettings(context.Background())) + require.JSONEq(t, `["X-Config-IP"]`, repo.updates[SettingKeyForwardedClientIPHeaders]) + require.Equal(t, []string{"X-Config-IP"}, cfg.ForwardedClientIPSettings().Headers) +} + +func TestSettingService_LoadForwardedClientIPSettingsReadFailureFailsClosed(t *testing.T) { repo := &forwardedIPMigrationRepoStub{ getMultipleErr: errors.New("database unavailable"), } @@ -650,13 +780,15 @@ func TestSettingService_LoadAPIKeyACLTrustForwardedIPSettingReadFailureFailsClos cfg.SetTrustForwardedIPForAPIKeyACL(true) svc := NewSettingService(repo, cfg) - err := svc.LoadAPIKeyACLTrustForwardedIPSetting(context.Background()) + err := svc.LoadForwardedClientIPSettings(context.Background()) require.ErrorContains(t, err, "get forwarded client ip settings") - require.False(t, cfg.TrustForwardedIPForAPIKeyACL()) + runtimeSettings := cfg.ForwardedClientIPSettings() + require.False(t, runtimeSettings.TrustForwardedIP) + require.Empty(t, runtimeSettings.Headers) } -func TestSettingService_LoadAPIKeyACLTrustForwardedIPSettingWriteFailureUsesComputedMode(t *testing.T) { +func TestSettingService_LoadForwardedClientIPSettingsWriteFailureUsesComputedMode(t *testing.T) { tests := []struct { name string trustedProxiesSet bool @@ -675,7 +807,7 @@ func TestSettingService_LoadAPIKeyACLTrustForwardedIPSettingWriteFailureUsesComp cfg := &config.Config{Server: config.ServerConfig{TrustedProxiesConfigured: test.trustedProxiesSet}} svc := NewSettingService(repo, cfg) - err := svc.LoadAPIKeyACLTrustForwardedIPSetting(context.Background()) + err := svc.LoadForwardedClientIPSettings(context.Background()) require.ErrorContains(t, err, "migrate forwarded client ip setting") require.Equal(t, test.wantEnabled, cfg.TrustForwardedIPForAPIKeyACL()) diff --git a/backend/internal/service/setting_update.go b/backend/internal/service/setting_update.go index 96c078e422..7449a3de69 100644 --- a/backend/internal/service/setting_update.go +++ b/backend/internal/service/setting_update.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/Wei-Shaw/sub2api/internal/config" "github.com/Wei-Shaw/sub2api/internal/pkg/antigravity" infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" ) @@ -62,6 +63,11 @@ func (s *SettingService) buildSystemSettingsUpdates(ctx context.Context, setting normalizedWhitelist = []string{} } settings.RegistrationEmailSuffixWhitelist = normalizedWhitelist + normalizedForwardedClientIPHeaders, err := config.NormalizeForwardedClientIPHeaders(settings.ForwardedClientIPHeaders) + if err != nil { + return nil, infraerrors.BadRequest("INVALID_FORWARDED_CLIENT_IP_HEADERS", err.Error()) + } + settings.ForwardedClientIPHeaders = normalizedForwardedClientIPHeaders alipaySource, err := normalizeVisibleMethodSettingSource("alipay", settings.PaymentVisibleMethodAlipaySource, settings.PaymentVisibleMethodAlipayEnabled) if err != nil { return nil, err @@ -156,6 +162,11 @@ func (s *SettingService) buildSystemSettingsUpdates(ctx context.Context, setting updates[SettingKeyTurnstileSecretKey] = settings.TurnstileSecretKey } updates[SettingKeyAPIKeyACLTrustForwardedIP] = strconv.FormatBool(settings.APIKeyACLTrustForwardedIP) + forwardedClientIPHeadersJSON, err := json.Marshal(settings.ForwardedClientIPHeaders) + if err != nil { + return nil, fmt.Errorf("marshal forwarded client IP headers: %w", err) + } + updates[SettingKeyForwardedClientIPHeaders] = string(forwardedClientIPHeadersJSON) // LinuxDo Connect OAuth 登录 updates[SettingKeyLinuxDoConnectEnabled] = strconv.FormatBool(settings.LinuxDoConnectEnabled) @@ -579,7 +590,7 @@ func (s *SettingService) refreshCachedSettings(settings *SystemSettings) { }) } if s.cfg != nil { - s.cfg.SetTrustForwardedIPForAPIKeyACL(settings.APIKeyACLTrustForwardedIP) + s.cfg.SetForwardedClientIPSettings(settings.APIKeyACLTrustForwardedIP, settings.ForwardedClientIPHeaders) } // codex_cli_only 加固策略缓存：设置更新后强制下次重载（涉及 4 个键 + JSON 解析，直接置过期）。 s.codexRestrictionPolicySF.Forget("codex_restriction_policy") diff --git a/backend/internal/service/wire.go b/backend/internal/service/wire.go index f3ee815394..76f40890ca 100644 --- a/backend/internal/service/wire.go +++ b/backend/internal/service/wire.go @@ -613,8 +613,8 @@ func ProvideSettingService(settingRepo SettingRepository, groupRepo GroupReposit svc := NewSettingService(settingRepo, cfg) svc.SetDefaultSubscriptionGroupReader(groupRepo) svc.SetProxyRepository(proxyRepo) - if err := svc.LoadAPIKeyACLTrustForwardedIPSetting(context.Background()); err != nil { - logger.LegacyPrintf("service.setting", "Warning: load api key acl forwarded ip setting failed: %v", err) + if err := svc.LoadForwardedClientIPSettings(context.Background()); err != nil { + logger.LegacyPrintf("service.setting", "Warning: load forwarded client IP settings failed: %v", err) } if err := svc.MigrateOpenAIAllowClaudeCodeCodexPluginSetting(context.Background()); err != nil { logger.LegacyPrintf("service.setting", "Warning: migrate openai allow Claude Code Codex plugin setting failed: %v", err) From 3c86e249f407c0ed886cb43ad7d7fbf05dcf3f29 Mon Sep 17 00:00:00 2001 From: Jlypx Date: Mon, 20 Jul 2026 00:09:21 +0800 Subject: [PATCH 47/59] =?UTF-8?q?feat:=20=E6=8E=A5=E5=85=A5=E5=AE=A2?= =?UTF-8?q?=E6=88=B7=E7=AB=AF=20IP=20=E8=AF=B7=E6=B1=82=E5=A4=B4=E7=AE=A1?= =?UTF-8?q?=E7=90=86=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .../internal/handler/admin/setting_handler.go | 1 + .../setting_handler_stepup_switch_test.go | 53 +++++++++++++++++++ .../handler/admin/setting_handler_update.go | 9 +++- backend/internal/handler/dto/settings.go | 9 ++-- 4 files changed, 67 insertions(+), 5 deletions(-) diff --git a/backend/internal/handler/admin/setting_handler.go b/backend/internal/handler/admin/setting_handler.go index d663cd61b3..6399c8eaf1 100644 --- a/backend/internal/handler/admin/setting_handler.go +++ b/backend/internal/handler/admin/setting_handler.go @@ -152,6 +152,7 @@ func (h *SettingHandler) GetSettings(c *gin.Context) { TurnstileSiteKey: settings.TurnstileSiteKey, TurnstileSecretKeyConfigured: settings.TurnstileSecretKeyConfigured, APIKeyACLTrustForwardedIP: settings.APIKeyACLTrustForwardedIP, + ForwardedClientIPHeaders: settings.ForwardedClientIPHeaders, LinuxDoConnectEnabled: settings.LinuxDoConnectEnabled, LinuxDoConnectClientID: settings.LinuxDoConnectClientID, LinuxDoConnectClientSecretConfigured: settings.LinuxDoConnectClientSecretConfigured, diff --git a/backend/internal/handler/admin/setting_handler_stepup_switch_test.go b/backend/internal/handler/admin/setting_handler_stepup_switch_test.go index 22f7ebf469..676252fc93 100644 --- a/backend/internal/handler/admin/setting_handler_stepup_switch_test.go +++ b/backend/internal/handler/admin/setting_handler_stepup_switch_test.go @@ -2,6 +2,7 @@ package admin import ( "bytes" + "context" "encoding/json" "net/http" "net/http/httptest" @@ -155,3 +156,55 @@ func TestUpdateSettingsOmittedSecuritySwitchesKeepDisabled(t *testing.T) { require.Equal(t, "false", repo.values[service.SettingKeyStepUpEnabled]) require.Equal(t, "false", repo.values[service.SettingKeySessionBindingEnabled]) } + +func TestUpdateSettingsForwardedClientIPHeadersOmittedPreservesAndEmptyClears(t *testing.T) { + h, repo := newStepUpSwitchTestHandler(t, map[string]string{ + service.SettingKeyForwardedClientIPHeaders: `["X-Cdn-Ip","True-Client-Ip"]`, + }) + + preserved := doUpdateSettings(t, h, map[string]any{"registration_enabled": true}, nil) + require.Equal(t, http.StatusOK, preserved.Code) + require.JSONEq(t, `["X-Cdn-Ip","True-Client-Ip"]`, repo.values[service.SettingKeyForwardedClientIPHeaders]) + require.Contains(t, preserved.Body.String(), `"forwarded_client_ip_headers":["X-Cdn-Ip","True-Client-Ip"]`) + + cleared := doUpdateSettings(t, h, map[string]any{"forwarded_client_ip_headers": []string{}}, nil) + require.Equal(t, http.StatusOK, cleared.Code) + require.JSONEq(t, `[]`, repo.values[service.SettingKeyForwardedClientIPHeaders]) + require.Contains(t, cleared.Body.String(), `"forwarded_client_ip_headers":[]`) +} + +func TestUpdateSettingsMalformedForwardedClientIPHeadersRemainFailClosedWhenOmitted(t *testing.T) { + cfg := &config.Config{Default: config.DefaultConfig{UserConcurrency: 5}} + repo := &settingHandlerRepoStub{values: map[string]string{ + service.SettingKeyAPIKeyACLTrustForwardedIP: "true", + service.SettingKeyForwardedClientIPHeaders: `{"not":"an array"}`, + }} + svc := service.NewSettingService(repo, cfg) + require.ErrorContains(t, svc.LoadForwardedClientIPSettings(context.Background()), "load forwarded client ip headers") + require.False(t, cfg.ForwardedClientIPSettings().TrustForwardedIP) + h := NewSettingHandler(svc, nil, nil, nil, nil, nil, nil) + + rec := doUpdateSettings(t, h, map[string]any{"registration_enabled": true}, nil) + + require.Equal(t, http.StatusOK, rec.Code) + require.Equal(t, "false", repo.values[service.SettingKeyAPIKeyACLTrustForwardedIP]) + require.JSONEq(t, `[]`, repo.values[service.SettingKeyForwardedClientIPHeaders]) + runtimeSettings := cfg.ForwardedClientIPSettings() + require.False(t, runtimeSettings.TrustForwardedIP) + require.Empty(t, runtimeSettings.Headers) + require.Contains(t, rec.Body.String(), `"api_key_acl_trust_forwarded_ip":false`) + require.Contains(t, rec.Body.String(), `"forwarded_client_ip_headers":[]`) +} + +func TestUpdateSettingsRejectsInvalidForwardedClientIPHeader(t *testing.T) { + h, repo := newStepUpSwitchTestHandler(t, map[string]string{ + service.SettingKeyForwardedClientIPHeaders: `["X-Existing-IP"]`, + }) + + rec := doUpdateSettings(t, h, map[string]any{ + "forwarded_client_ip_headers": []string{"X Invalid"}, + }, nil) + + require.Equal(t, http.StatusBadRequest, rec.Code) + require.JSONEq(t, `["X-Existing-IP"]`, repo.values[service.SettingKeyForwardedClientIPHeaders]) +} diff --git a/backend/internal/handler/admin/setting_handler_update.go b/backend/internal/handler/admin/setting_handler_update.go index 40343ed311..2f5792e213 100644 --- a/backend/internal/handler/admin/setting_handler_update.go +++ b/backend/internal/handler/admin/setting_handler_update.go @@ -51,7 +51,8 @@ type UpdateSettingsRequest struct { TurnstileSecretKey string `json:"turnstile_secret_key"` // API Key IP 访问控制设置 - APIKeyACLTrustForwardedIP *bool `json:"api_key_acl_trust_forwarded_ip"` + APIKeyACLTrustForwardedIP *bool `json:"api_key_acl_trust_forwarded_ip"` + ForwardedClientIPHeaders *[]string `json:"forwarded_client_ip_headers"` // LinuxDo Connect OAuth 登录 LinuxDoConnectEnabled bool `json:"linuxdo_connect_enabled"` @@ -400,6 +401,10 @@ func (h *SettingHandler) UpdateSettings(c *gin.Context) { if req.StepUpEnabled != nil { stepUpEnabled = *req.StepUpEnabled } + forwardedClientIPHeaders := append([]string(nil), previousSettings.ForwardedClientIPHeaders...) + if req.ForwardedClientIPHeaders != nil { + forwardedClientIPHeaders = append([]string(nil), (*req.ForwardedClientIPHeaders)...) + } // 开启敏感操作 step-up 门控属自锁风险操作：仅允许本人已启用 TOTP 的管理员会话开启， // 否则开启后操作者立即被挡在所有敏感操作之外。仅在 false→true 的开启瞬间校验， @@ -1269,6 +1274,7 @@ func (h *SettingHandler) UpdateSettings(c *gin.Context) { } return previousSettings.APIKeyACLTrustForwardedIP }(), + ForwardedClientIPHeaders: forwardedClientIPHeaders, LinuxDoConnectEnabled: req.LinuxDoConnectEnabled, LinuxDoConnectClientID: req.LinuxDoConnectClientID, LinuxDoConnectClientSecret: req.LinuxDoConnectClientSecret, @@ -1796,6 +1802,7 @@ func (h *SettingHandler) UpdateSettings(c *gin.Context) { TurnstileSiteKey: updatedSettings.TurnstileSiteKey, TurnstileSecretKeyConfigured: updatedSettings.TurnstileSecretKeyConfigured, APIKeyACLTrustForwardedIP: updatedSettings.APIKeyACLTrustForwardedIP, + ForwardedClientIPHeaders: updatedSettings.ForwardedClientIPHeaders, LinuxDoConnectEnabled: updatedSettings.LinuxDoConnectEnabled, LinuxDoConnectClientID: updatedSettings.LinuxDoConnectClientID, LinuxDoConnectClientSecretConfigured: updatedSettings.LinuxDoConnectClientSecretConfigured, diff --git a/backend/internal/handler/dto/settings.go b/backend/internal/handler/dto/settings.go index fd7e7b6a52..a62b9b07a7 100644 --- a/backend/internal/handler/dto/settings.go +++ b/backend/internal/handler/dto/settings.go @@ -52,10 +52,11 @@ type SystemSettings struct { SMTPFromName string `json:"smtp_from_name"` SMTPUseTLS bool `json:"smtp_use_tls"` - TurnstileEnabled bool `json:"turnstile_enabled"` - TurnstileSiteKey string `json:"turnstile_site_key"` - TurnstileSecretKeyConfigured bool `json:"turnstile_secret_key_configured"` - APIKeyACLTrustForwardedIP bool `json:"api_key_acl_trust_forwarded_ip"` + TurnstileEnabled bool `json:"turnstile_enabled"` + TurnstileSiteKey string `json:"turnstile_site_key"` + TurnstileSecretKeyConfigured bool `json:"turnstile_secret_key_configured"` + APIKeyACLTrustForwardedIP bool `json:"api_key_acl_trust_forwarded_ip"` + ForwardedClientIPHeaders []string `json:"forwarded_client_ip_headers"` LinuxDoConnectEnabled bool `json:"linuxdo_connect_enabled"` LinuxDoConnectClientID string `json:"linuxdo_connect_client_id"` From fedeba2568b951206cd41b980bbc1111a0e0cf52 Mon Sep 17 00:00:00 2001 From: Jlypx Date: Mon, 20 Jul 2026 00:09:37 +0800 Subject: [PATCH 48/59] =?UTF-8?q?feat:=20=E5=AE=A1=E8=AE=A1=E5=AE=A2?= =?UTF-8?q?=E6=88=B7=E7=AB=AF=20IP=20=E8=AF=B7=E6=B1=82=E5=A4=B4=E5=8F=98?= =?UTF-8?q?=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- backend/internal/handler/admin/setting_handler_audit.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/internal/handler/admin/setting_handler_audit.go b/backend/internal/handler/admin/setting_handler_audit.go index 7d3a420ff4..c8b8bf1b24 100644 --- a/backend/internal/handler/admin/setting_handler_audit.go +++ b/backend/internal/handler/admin/setting_handler_audit.go @@ -107,6 +107,9 @@ func diffSettings(before *service.SystemSettings, after *service.SystemSettings, if before.APIKeyACLTrustForwardedIP != after.APIKeyACLTrustForwardedIP { changed = append(changed, "api_key_acl_trust_forwarded_ip") } + if !equalStringSlice(before.ForwardedClientIPHeaders, after.ForwardedClientIPHeaders) { + changed = append(changed, "forwarded_client_ip_headers") + } if before.LinuxDoConnectEnabled != after.LinuxDoConnectEnabled { changed = append(changed, "linuxdo_connect_enabled") } From ff5b0e6254a81247e4a16e0728aa8cf5ce0937b1 Mon Sep 17 00:00:00 2001 From: Jlypx Date: Mon, 20 Jul 2026 00:09:47 +0800 Subject: [PATCH 49/59] =?UTF-8?q?test:=20=E6=9B=B4=E6=96=B0=E5=AE=A2?= =?UTF-8?q?=E6=88=B7=E7=AB=AF=20IP=20=E8=AE=BE=E7=BD=AE=E5=93=8D=E5=BA=94?= =?UTF-8?q?=E5=A5=91=E7=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- backend/internal/server/api_contract_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/internal/server/api_contract_test.go b/backend/internal/server/api_contract_test.go index 86ac46c2af..bffce67cd6 100644 --- a/backend/internal/server/api_contract_test.go +++ b/backend/internal/server/api_contract_test.go @@ -791,6 +791,7 @@ func TestAPIContracts(t *testing.T) { "site_subtitle": "Subtitle", "api_base_url": "https://api.example.com", "api_key_acl_trust_forwarded_ip": false, + "forwarded_client_ip_headers": [], "contact_info": "support", "doc_url": "https://docs.example.com", "auth_source_default_email_balance": 0, @@ -1102,6 +1103,7 @@ func TestAPIContracts(t *testing.T) { "site_subtitle": "Subscription to API Conversion Platform", "api_base_url": "", "api_key_acl_trust_forwarded_ip": false, + "forwarded_client_ip_headers": [], "contact_info": "", "doc_url": "", "home_content": "", From 344f3039196c61c114ae73013dbf09c95abc22bb Mon Sep 17 00:00:00 2001 From: Jlypx Date: Mon, 20 Jul 2026 00:09:57 +0800 Subject: [PATCH 50/59] =?UTF-8?q?feat:=20=E6=89=A9=E5=B1=95=E5=89=8D?= =?UTF-8?q?=E7=AB=AF=E5=AE=A2=E6=88=B7=E7=AB=AF=20IP=20=E8=AE=BE=E7=BD=AE?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- frontend/src/api/admin/settings.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/api/admin/settings.ts b/frontend/src/api/admin/settings.ts index 33b1f7f97d..2779c2db62 100644 --- a/frontend/src/api/admin/settings.ts +++ b/frontend/src/api/admin/settings.ts @@ -455,6 +455,7 @@ export interface SystemSettings { turnstile_site_key: string; turnstile_secret_key_configured: boolean; api_key_acl_trust_forwarded_ip: boolean; + forwarded_client_ip_headers: string[]; // LinuxDo Connect OAuth settings linuxdo_connect_enabled: boolean; @@ -757,6 +758,7 @@ export interface UpdateSettingsRequest { turnstile_site_key?: string; turnstile_secret_key?: string; api_key_acl_trust_forwarded_ip?: boolean; + forwarded_client_ip_headers?: string[]; linuxdo_connect_enabled?: boolean; linuxdo_connect_client_id?: string; linuxdo_connect_client_secret?: string; From 4990ca0b2cb5b253a65fc0e0cf4dfc74040084c1 Mon Sep 17 00:00:00 2001 From: Jlypx Date: Mon, 20 Jul 2026 00:10:05 +0800 Subject: [PATCH 51/59] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E5=AE=A2?= =?UTF-8?q?=E6=88=B7=E7=AB=AF=20IP=20=E8=AF=B7=E6=B1=82=E5=A4=B4=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E7=95=8C=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- frontend/src/views/admin/SettingsView.vue | 211 ++++++++++++++++++ .../admin/__tests__/SettingsView.spec.ts | 33 ++- 2 files changed, 243 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/admin/SettingsView.vue b/frontend/src/views/admin/SettingsView.vue index c5c898ee85..bf749bb73b 100644 --- a/frontend/src/views/admin/SettingsView.vue +++ b/frontend/src/views/admin/SettingsView.vue @@ -1654,6 +1654,66 @@
+ +
+ +

+ {{ t("admin.settings.apiKeyAcl.forwardedClientIpHeadersHint") }} +

+
+
+ + {{ header }} + + +
+ +
+
+
+

+ {{ t("admin.settings.apiKeyAcl.forwardedClientIpHeadersRiskHint") }} +

+
@@ -7754,6 +7814,7 @@ const smtpPasswordManuallyEdited = ref(false); const testEmailAddress = ref(""); const registrationEmailSuffixWhitelistTags = ref([]); const registrationEmailSuffixWhitelistDraft = ref(""); +const forwardedClientIpHeaderDraft = ref(""); const tablePageSizeOptionsInput = ref("10, 20, 50, 100"); // Admin API Key 状态 @@ -8394,6 +8455,7 @@ const form = reactive({ turnstile_secret_key: "", turnstile_secret_key_configured: false, api_key_acl_trust_forwarded_ip: true, + forwarded_client_ip_headers: [], // LinuxDo Connect OAuth 登录 linuxdo_connect_enabled: false, linuxdo_connect_client_id: "", @@ -8972,6 +9034,143 @@ function handleRegistrationEmailSuffixWhitelistPaste(event: ClipboardEvent) { } } +const forwardedClientIpHeaderSeparatorKeys = new Set([ + " ", + ",", + "，", + "Enter", + "Tab", +]); +const forwardedClientIpHeaderTokenPattern = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/; +const maxForwardedClientIpHeaders = 16; + +type ForwardedClientIpHeaderResult = "added" | "duplicate" | "invalid" | "full"; + +function normalizeForwardedClientIpHeader(raw: string): string { + const header = raw.trim(); + if (!forwardedClientIpHeaderTokenPattern.test(header)) { + return ""; + } + + return header + .toLowerCase() + .split("-") + .map((part) => `${part.charAt(0).toUpperCase()}${part.slice(1)}`) + .join("-"); +} + +function normalizeForwardedClientIpHeaders(value: unknown): string[] { + if (!Array.isArray(value)) { + return []; + } + + const headers: string[] = []; + const seen = new Set(); + for (const raw of value) { + if (typeof raw !== "string") { + continue; + } + const header = normalizeForwardedClientIpHeader(raw); + const key = header.toLowerCase(); + if (!header || seen.has(key) || headers.length >= maxForwardedClientIpHeaders) { + continue; + } + seen.add(key); + headers.push(header); + } + return headers; +} + +function removeForwardedClientIpHeader(header: string) { + form.forwarded_client_ip_headers = form.forwarded_client_ip_headers.filter( + (item) => item !== header, + ); +} + +function addForwardedClientIpHeader(raw: string): ForwardedClientIpHeaderResult { + const header = normalizeForwardedClientIpHeader(raw); + if (!header) { + return "invalid"; + } + if ( + form.forwarded_client_ip_headers.some( + (item) => item.toLowerCase() === header.toLowerCase(), + ) + ) { + return "duplicate"; + } + if (form.forwarded_client_ip_headers.length >= maxForwardedClientIpHeaders) { + return "full"; + } + form.forwarded_client_ip_headers = [ + ...form.forwarded_client_ip_headers, + header, + ]; + return "added"; +} + +function showForwardedClientIpHeaderError(result: ForwardedClientIpHeaderResult) { + if (result === "invalid") { + appStore.showError(t("admin.settings.apiKeyAcl.forwardedClientIpHeaderInvalid")); + } else if (result === "full") { + appStore.showError( + t("admin.settings.apiKeyAcl.forwardedClientIpHeadersLimit", { + max: maxForwardedClientIpHeaders, + }), + ); + } +} + +function commitForwardedClientIpHeaderDraft() { + const draft = forwardedClientIpHeaderDraft.value; + if (!draft) { + return; + } + const result = addForwardedClientIpHeader(draft); + showForwardedClientIpHeaderError(result); + forwardedClientIpHeaderDraft.value = ""; +} + +function handleForwardedClientIpHeaderKeydown(event: KeyboardEvent) { + if (event.isComposing) { + return; + } + if (forwardedClientIpHeaderSeparatorKeys.has(event.key)) { + event.preventDefault(); + commitForwardedClientIpHeaderDraft(); + return; + } + if ( + event.key === "Backspace" && + !forwardedClientIpHeaderDraft.value && + form.forwarded_client_ip_headers.length > 0 + ) { + form.forwarded_client_ip_headers.pop(); + } +} + +function handleForwardedClientIpHeaderPaste(event: ClipboardEvent) { + const text = event.clipboardData?.getData("text") || ""; + if (!text.trim()) { + return; + } + event.preventDefault(); + + let error: ForwardedClientIpHeaderResult | undefined; + for (const token of text.split(/[,，;\r\n]+/)) { + if (!token.trim()) { + continue; + } + const result = addForwardedClientIpHeader(token); + if (result === "invalid" || result === "full") { + error = result; + } + } + if (error) { + showForwardedClientIpHeaderError(error); + } +} + // Quota notify email helpers const addQuotaNotifyEmail = () => { if (!form.account_quota_notify_emails) { @@ -9354,6 +9553,10 @@ async function loadSettings() { normalizeRegistrationEmailSuffixDomains( settings.registration_email_suffix_whitelist, ); + form.forwarded_client_ip_headers = normalizeForwardedClientIpHeaders( + settings.forwarded_client_ip_headers, + ); + forwardedClientIpHeaderDraft.value = ""; tablePageSizeOptionsInput.value = formatTablePageSizeOptions( Array.isArray(settings.table_page_size_options) ? settings.table_page_size_options @@ -9595,6 +9798,9 @@ async function saveSettings() { form.login_agreement_mode = form.login_agreement_mode === "checkbox" ? "checkbox" : "modal"; form.login_agreement_documents = normalizedLoginAgreementDocuments; + form.forwarded_client_ip_headers = normalizeForwardedClientIpHeaders( + form.forwarded_client_ip_headers, + ); const normalizedDefaultSubscriptions = normalizeDefaultSubscriptionSettings( form.default_subscriptions, @@ -9728,6 +9934,7 @@ async function saveSettings() { turnstile_site_key: form.turnstile_site_key, turnstile_secret_key: form.turnstile_secret_key || undefined, api_key_acl_trust_forwarded_ip: form.api_key_acl_trust_forwarded_ip, + forwarded_client_ip_headers: form.forwarded_client_ip_headers, linuxdo_connect_enabled: form.linuxdo_connect_enabled, linuxdo_connect_client_id: form.linuxdo_connect_client_id, linuxdo_connect_client_secret: @@ -9993,6 +10200,10 @@ async function saveSettings() { normalizeRegistrationEmailSuffixDomains( updated.registration_email_suffix_whitelist, ); + form.forwarded_client_ip_headers = normalizeForwardedClientIpHeaders( + updated.forwarded_client_ip_headers, + ); + forwardedClientIpHeaderDraft.value = ""; tablePageSizeOptionsInput.value = formatTablePageSizeOptions( Array.isArray(updated.table_page_size_options) ? updated.table_page_size_options diff --git a/frontend/src/views/admin/__tests__/SettingsView.spec.ts b/frontend/src/views/admin/__tests__/SettingsView.spec.ts index 8b7ffb301c..e913a1249a 100644 --- a/frontend/src/views/admin/__tests__/SettingsView.spec.ts +++ b/frontend/src/views/admin/__tests__/SettingsView.spec.ts @@ -368,6 +368,7 @@ const baseSettingsResponse = { turnstile_site_key: "", turnstile_secret_key_configured: false, api_key_acl_trust_forwarded_ip: true, + forwarded_client_ip_headers: [], linuxdo_connect_enabled: false, linuxdo_connect_client_id: "", linuxdo_connect_client_secret_configured: false, @@ -653,10 +654,11 @@ describe("admin SettingsView payment visible method controls", () => { expect(wrapper.text()).not.toContain("支付来源"); }); - it("loads and saves the forwarded client IP takeover switch", async () => { + it("loads, edits, validates, and saves forwarded client-IP headers", async () => { getSettings.mockResolvedValueOnce({ ...baseSettingsResponse, api_key_acl_trust_forwarded_ip: false, + forwarded_client_ip_headers: ["cf-connecting-ip", "X-Real-IP"], }); const wrapper = mountView(); @@ -669,14 +671,43 @@ describe("admin SettingsView payment visible method controls", () => { expect(card).toBeDefined(); const toggle = card!.get('input[type="checkbox"]'); expect((toggle.element as HTMLInputElement).checked).toBe(false); + expect(card!.find('[data-testid="forwarded-client-ip-headers-input"]').exists()).toBe(false); await toggle.setValue(true); + expect(card!.findAll('[data-testid="forwarded-client-ip-header-tag"]')).toHaveLength(2); + expect(card!.text()).toContain("Cf-Connecting-Ip"); + expect(card!.text()).toContain("X-Real-Ip"); + showError.mockClear(); + + const input = card!.get('[data-testid="forwarded-client-ip-headers-input"]'); + await input.setValue("x-client-ip"); + await input.trigger("keydown", { key: "Enter" }); + await input.setValue("X-CLIENT-IP"); + await input.trigger("keydown", { key: "Enter" }); + await input.setValue("invalid header"); + await input.trigger("keydown", { key: "Enter" }); + expect(showError).toHaveBeenCalledTimes(1); + expect(card!.findAll('[data-testid="forwarded-client-ip-header-tag"]')).toHaveLength(3); + + const realIpTag = card! + .findAll('[data-testid="forwarded-client-ip-header-tag"]') + .find((tag) => tag.text().includes("X-Real-Ip")); + expect(realIpTag).toBeDefined(); + await realIpTag!.get("button").trigger("click"); + expect(card!.text()).not.toContain("X-Real-Ip"); + + await toggle.setValue(false); + expect(card!.find('[data-testid="forwarded-client-ip-headers-input"]').exists()).toBe(false); + await toggle.setValue(true); + expect(card!.text()).toContain("X-Client-Ip"); + await wrapper.find("form").trigger("submit.prevent"); await flushPromises(); expect(updateSettings).toHaveBeenCalledWith( expect.objectContaining({ api_key_acl_trust_forwarded_ip: true, + forwarded_client_ip_headers: ["Cf-Connecting-Ip", "X-Client-Ip"], }), ); }); From 80dd2929a309a99c7a3e6ed72ac4312df2f5c9a2 Mon Sep 17 00:00:00 2001 From: Jlypx Date: Mon, 20 Jul 2026 00:10:18 +0800 Subject: [PATCH 52/59] =?UTF-8?q?docs:=20=E6=9C=AC=E5=9C=B0=E5=8C=96?= =?UTF-8?q?=E5=AE=A2=E6=88=B7=E7=AB=AF=20IP=20=E8=AF=B7=E6=B1=82=E5=A4=B4?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- frontend/src/i18n/locales/en/admin/settings.ts | 9 ++++++++- frontend/src/i18n/locales/zh/admin/settings.ts | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/frontend/src/i18n/locales/en/admin/settings.ts b/frontend/src/i18n/locales/en/admin/settings.ts index c08ca2ae52..ec68ce1b10 100644 --- a/frontend/src/i18n/locales/en/admin/settings.ts +++ b/frontend/src/i18n/locales/en/admin/settings.ts @@ -153,7 +153,14 @@ export default { 'Choose which client IP is used by API Key allowlists/denylists, admin audit logs, and session IP/UA binding', trustForwardedIp: 'Trust forwarded client IP', trustForwardedIpHint: - 'Enabled by default for upgrade compatibility. When enabled, raw CF-Connecting-IP, X-Real-IP, or X-Forwarded-For values take over server.trusted_proxies for client-IP resolution. Disable it to enforce the Gin trusted-proxy chain configured by server.trusted_proxies. Only enable takeover mode when the origin cannot be reached directly. Changing this switch changes existing session IP fingerprints.' + 'Enabled by default for upgrade compatibility. When enabled, raw CF-Connecting-IP, X-Real-IP, or X-Forwarded-For values take over server.trusted_proxies for client-IP resolution. Disable it to enforce the Gin trusted-proxy chain configured by server.trusted_proxies. Only enable takeover mode when the origin cannot be reached directly. Changing this switch changes existing session IP fingerprints.', + forwardedClientIpHeaders: 'Custom client-IP headers', + forwardedClientIpHeadersHint: 'Add CDN or proxy header names to check before the built-in headers.', + forwardedClientIpHeadersPlaceholder: 'X-Client-IP', + forwardedClientIpHeadersRiskHint: 'These raw headers can be spoofed when the origin is reachable directly. Restrict origin access before trusting them.', + forwardedClientIpHeaderInvalid: 'Enter a valid HTTP header name.', + forwardedClientIpHeadersLimit: 'At most {max} custom client-IP headers are allowed.', + removeForwardedClientIpHeader: 'Remove {header}' }, linuxdo: { title: 'LinuxDo Connect Login', diff --git a/frontend/src/i18n/locales/zh/admin/settings.ts b/frontend/src/i18n/locales/zh/admin/settings.ts index bfd3f139d5..f5827d0e1b 100644 --- a/frontend/src/i18n/locales/zh/admin/settings.ts +++ b/frontend/src/i18n/locales/zh/admin/settings.ts @@ -152,7 +152,14 @@ export default { description: '控制 API Key 白/黑名单、操作审计日志与会话 IP/UA 绑定使用哪个客户端 IP 判断', trustForwardedIp: '信任反代传递的客户端 IP', trustForwardedIpHint: - '为保证升级兼容默认开启。开启后 CF-Connecting-IP、X-Real-IP 或 X-Forwarded-For 会直接接管客户端 IP 解析并覆盖 server.trusted_proxies；关闭后严格使用 server.trusted_proxies 配置的 Gin 可信代理链。仅在源站无法被直接访问时开启接管模式。切换会改变现有会话的 IP 指纹。' + '为保证升级兼容默认开启。开启后 CF-Connecting-IP、X-Real-IP 或 X-Forwarded-For 会直接接管客户端 IP 解析并覆盖 server.trusted_proxies；关闭后严格使用 server.trusted_proxies 配置的 Gin 可信代理链。仅在源站无法被直接访问时开启接管模式。切换会改变现有会话的 IP 指纹。', + forwardedClientIpHeaders: '自定义客户端 IP 请求头', + forwardedClientIpHeadersHint: '添加 CDN 或反代请求头名称，解析时优先于内置请求头。', + forwardedClientIpHeadersPlaceholder: 'X-Client-IP', + forwardedClientIpHeadersRiskHint: '源站可被直接访问时，这些原始请求头可被伪造；请先限制源站访问再信任它们。', + forwardedClientIpHeaderInvalid: '请输入有效的 HTTP 请求头名称。', + forwardedClientIpHeadersLimit: '自定义客户端 IP 请求头最多允许 {max} 个。', + removeForwardedClientIpHeader: '移除 {header}' }, linuxdo: { title: 'LinuxDo Connect 登录', From 6becd11e3915577e5624e8651897a6d7aa821c7f Mon Sep 17 00:00:00 2001 From: Jlypx Date: Mon, 20 Jul 2026 00:10:32 +0800 Subject: [PATCH 53/59] =?UTF-8?q?docs:=20=E6=9B=B4=E6=96=B0=E5=AE=A2?= =?UTF-8?q?=E6=88=B7=E7=AB=AF=20IP=20=E8=BE=B9=E7=BC=98=E5=AE=89=E5=85=A8?= =?UTF-8?q?=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- deploy/EDGE_SECURITY.md | 45 ++++++++++++++++++++++++++++---------- deploy/config.example.yaml | 11 ++++++++++ 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/deploy/EDGE_SECURITY.md b/deploy/EDGE_SECURITY.md index 0c0e78768a..956507c4fb 100644 --- a/deploy/EDGE_SECURITY.md +++ b/deploy/EDGE_SECURITY.md @@ -30,20 +30,41 @@ the application's responsibility. ## Trusted client IPs `security.trust_forwarded_ip_for_api_key_acl` is enabled by default for upgrade -compatibility. On the first upgrade to this mode, a legacy `false` value is -changed to `true` only when `server.trusted_proxies` was not explicitly -configured; explicit proxy policies remain in secure mode. Later administrator -changes are preserved. While enabled, raw `CF-Connecting-IP`, `X-Real-IP`, and -`X-Forwarded-For` values take over client-IP resolution for logs and -security-sensitive paths. Disable the switch to make Gin's -`server.trusted_proxies` chain authoritative. Configure only the exact CIDR/IP -addresses that connect directly to Sub2API; an explicit empty list trusts no -forwarded client IPs while the switch is disabled. +compatibility. While enabled, raw forwarding headers take over client-IP +resolution for logs and security-sensitive paths. Custom headers from +`security.forwarded_client_ip_headers` are checked in configured order before +the built-in `CF-Connecting-IP`, `X-Real-IP`, and `X-Forwarded-For` fallback. +Header names are case-insensitive, normalized when loaded, de-duplicated, and +limited to 16 unique valid HTTP field names. Header values must contain IP +literals; comma-separated values are supported, invalid entries are skipped, +and public addresses are preferred over private fallback addresses. + +The list can be supplied in YAML or with the comma-separated environment +variable `SECURITY_FORWARDED_CLIENT_IP_HEADERS`; an explicitly empty environment +value clears YAML values. It is also editable from the admin security settings +and updates at runtime without a restart. A request snapshots the switch and +header list together, so one request cannot mix old and new settings. Custom +headers are ignored completely when the switch is disabled. In that mode Gin's +`server.trusted_proxies` chain is authoritative: configure only the exact +CIDR/IP addresses that connect directly to Sub2API. An explicit empty list +trusts no forwarded client IPs. + +On the first upgrade to this mode, a legacy `false` value is changed to `true` +only when `server.trusted_proxies` was not explicitly configured; explicit +proxy policies remain in secure mode. New installations persist the configured +custom header list during database initialization. Existing installations +backfill a missing database value from the YAML configuration. A hidden +migration marker prevents later administrator changes from being overwritten. +If settings cannot be read or the persisted custom-header list is malformed, +the process fails closed to trusted-proxy mode with no custom headers. If a +migration write fails, the computed mode remains active for the current process +and startup records a warning. Compatibility takeover accepts forwarded headers without validating the direct -peer. Protect the origin from direct access while it is enabled. A CDN -deployment must firewall the origin so only the CDN or load balancer can reach -it, and the proxy must overwrite forwarded headers. +peer, including any configured custom header. Protect the origin from direct +access while it is enabled. A CDN deployment must firewall the origin so only +the CDN or load balancer can reach it, and that proxy must overwrite every +trusted client-IP header rather than append an untrusted client value. Example for a proxy on the same host: diff --git a/deploy/config.example.yaml b/deploy/config.example.yaml index e84a906d4d..c0fb225ee9 100644 --- a/deploy/config.example.yaml +++ b/deploy/config.example.yaml @@ -106,6 +106,17 @@ security: # 旧版兼容开关。开启时原始转发头会接管 server.trusted_proxies；关闭后严格 # 使用上方配置的可信代理链。示例配置采用高安全模式。 trust_forwarded_ip_for_api_key_acl: false + # Optional client-IP headers for third-party CDNs, checked in list order before + # CF-Connecting-IP, X-Real-IP, and X-Forwarded-For. Valid only while the legacy + # compatibility switch above is true; maximum 16 unique HTTP header names. + # 第三方 CDN 的可选客户端 IP 请求头，按列表顺序优先于内置请求头解析。 + # 仅在上方旧版兼容开关为 true 时生效；最多配置 16 个不重复的合法 HTTP 头名。 + # Environment / 环境变量: SECURITY_FORWARDED_CLIENT_IP_HEADERS=True-Client-IP,X-CDN-Client-IP + forwarded_client_ip_headers: [] + # Example / 示例: + # forwarded_client_ip_headers: + # - "True-Client-IP" + # - "X-CDN-Client-IP" url_allowlist: # Enable URL allowlist validation (disable to skip all URL checks) # 启用 URL 白名单验证（禁用则跳过所有 URL 检查） From 0cc382e3317d8b9de1eb66e669e756ff45e0327d Mon Sep 17 00:00:00 2001 From: Jlypx Date: Mon, 20 Jul 2026 00:10:41 +0800 Subject: [PATCH 54/59] =?UTF-8?q?docs:=20=E8=A1=A5=E5=85=85=E5=AE=A2?= =?UTF-8?q?=E6=88=B7=E7=AB=AF=20IP=20=E8=AF=B7=E6=B1=82=E5=A4=B4=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- README.md | 9 +++++++++ README_CN.md | 9 +++++++++ README_JA.md | 9 +++++++++ 3 files changed, 27 insertions(+) diff --git a/README.md b/README.md index 16e7bd9a5d..1286413b9f 100644 --- a/README.md +++ b/README.md @@ -572,8 +572,17 @@ Additional security-related options are available in `config.yaml`: - `security.csp` to control Content-Security-Policy headers - `billing.circuit_breaker` to fail closed on billing errors - `security.trust_forwarded_ip_for_api_key_acl` enables legacy raw forwarded-header takeover (enabled by default for upgrade compatibility); disable it to enforce `server.trusted_proxies`, which should contain only the exact proxy CIDRs that connect directly to Sub2API +- `security.forwarded_client_ip_headers` configures up to 16 third-party CDN client-IP header names; they are checked in order before the built-in headers only while legacy takeover is enabled - `turnstile.required` to require Turnstile in release mode +Custom client-IP headers can be set in YAML or as a comma-separated environment variable: + +```bash +SECURITY_FORWARDED_CLIENT_IP_HEADERS=True-Client-IP,X-CDN-Client-IP +``` + +Header names are validated, canonicalized, and de-duplicated. The admin security settings can update the list without a restart; new installations persist YAML/environment defaults and existing installations backfill a missing database value. When legacy takeover is disabled, all custom and built-in raw forwarding headers are ignored and Gin uses only `server.trusted_proxies`. While takeover is enabled, firewall the origin to CDN/proxy addresses and make the edge overwrite every trusted client-IP header. See [`deploy/EDGE_SECURITY.md`](deploy/EDGE_SECURITY.md) for the complete migration and trust-boundary rules. + **⚠️ Security Warning: HTTP URL Configuration** When `security.url_allowlist.enabled=false`, the system performs minimal URL validation and **allows HTTP URLs by default** (dev-friendly mode; Docker Compose deployments use the same default). For production, explicitly tighten this to HTTPS-only: diff --git a/README_CN.md b/README_CN.md index a9aea3bbe8..b42d5516a0 100644 --- a/README_CN.md +++ b/README_CN.md @@ -608,8 +608,17 @@ gateway: - `security.csp` 配置 Content-Security-Policy - `billing.circuit_breaker` 计费异常时 fail-closed - `security.trust_forwarded_ip_for_api_key_acl` 控制旧版原始转发头接管（为升级兼容默认开启）；关闭后严格使用 `server.trusted_proxies`，其中只应填写直接连接 Sub2API 的精确代理 CIDR +- `security.forwarded_client_ip_headers` 最多配置 16 个第三方 CDN 客户端 IP 请求头；仅在旧版接管开启时按顺序优先于内置请求头解析 - `turnstile.required` 在 release 模式强制启用 Turnstile +自定义客户端 IP 请求头可通过 YAML 配置，也可使用逗号分隔的环境变量： + +```bash +SECURITY_FORWARDED_CLIENT_IP_HEADERS=True-Client-IP,X-CDN-Client-IP +``` + +请求头名称会经过合法性校验、规范化和大小写无关去重。管理员可在安全设置中动态更新列表，无需重启；新安装会持久化 YAML/环境变量默认值，旧安装缺少数据库字段时会自动回填。关闭旧版接管后，自定义头和内置原始转发头均被忽略，只使用 `server.trusted_proxies`。开启接管时必须限制源站仅允许 CDN/代理访问，并确保边缘代理覆盖所有受信客户端 IP 请求头。完整迁移规则和信任边界见 [`deploy/EDGE_SECURITY.md`](deploy/EDGE_SECURITY.md)。 + **网关防御纵深建议（重点）** - `gateway.upstream_response_read_max_bytes`：限制非流式上游响应读取大小（默认 `8MB`），用于防止异常响应导致内存放大。 diff --git a/README_JA.md b/README_JA.md index 8d95162612..37ad6a2dc4 100644 --- a/README_JA.md +++ b/README_JA.md @@ -570,8 +570,17 @@ default: - `security.csp` - Content-Security-Policy ヘッダーの制御 - `billing.circuit_breaker` - 課金エラー時にフェイルクローズ - `security.trust_forwarded_ip_for_api_key_acl` - 従来の生転送ヘッダーによる上書きを制御（アップグレード互換性のため既定で有効）。無効にすると `server.trusted_proxies` を厳格に使用し、Sub2API に直接接続するプロキシの正確な CIDR のみを指定 +- `security.forwarded_client_ip_headers` - サードパーティ CDN のクライアント IP ヘッダーを最大 16 個指定。従来モードが有効な場合のみ、設定順で組み込みヘッダーより先に評価 - `turnstile.required` - リリースモードでの Turnstile 必須化 +カスタムクライアント IP ヘッダーは YAML またはカンマ区切りの環境変数で設定できます: + +```bash +SECURITY_FORWARDED_CLIENT_IP_HEADERS=True-Client-IP,X-CDN-Client-IP +``` + +ヘッダー名は検証、正規化、大小文字を区別しない重複排除が行われます。管理画面のセキュリティ設定から再起動せずに更新でき、新規インストールでは YAML/環境変数の既定値を保存し、既存環境ではデータベース値がない場合に補完します。従来モードを無効にするとカスタムおよび組み込みの生転送ヘッダーはすべて無視され、`server.trusted_proxies` のみを使用します。有効にする場合はオリジンへの接続元を CDN/プロキシに制限し、エッジで信頼する全クライアント IP ヘッダーを上書きしてください。移行規則と信頼境界の詳細は [`deploy/EDGE_SECURITY.md`](deploy/EDGE_SECURITY.md) を参照してください。 + **⚠️ セキュリティ警告: HTTP URL 設定** `security.url_allowlist.enabled=false` の場合、システムは最小限の URL バリデーションのみを行い、**デフォルトで HTTP URL を許可**します（開発フレンドリーモード。Docker Compose デプロイのデフォルトも同じです）。本番環境では、以下のように明示的に HTTPS のみに制限することを推奨します: From dd0cbe91c9c847b541d199c30cc3d08f4ee9fe04 Mon Sep 17 00:00:00 2001 From: Jlypx Date: Mon, 20 Jul 2026 00:22:45 +0800 Subject: [PATCH 55/59] =?UTF-8?q?fix:=20=E9=81=BF=E5=85=8D=E5=A4=8D?= =?UTF-8?q?=E5=88=B6=E5=AE=A2=E6=88=B7=E7=AB=AF=20IP=20=E5=8E=9F=E5=AD=90?= =?UTF-8?q?=E7=8A=B6=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- backend/internal/config/config.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index c60ef2a461..c5165c7c05 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -685,9 +685,9 @@ type SecurityConfig struct { ProxyProbe ProxyProbeConfig `mapstructure:"proxy_probe"` // TrustForwardedIPForAPIKeyACL enables legacy raw forwarded-header takeover. // When disabled, server.trusted_proxies is authoritative for all client-IP consumers. - TrustForwardedIPForAPIKeyACL bool `mapstructure:"trust_forwarded_ip_for_api_key_acl"` - ForwardedClientIPHeaders []string `mapstructure:"forwarded_client_ip_headers" json:"forwarded_client_ip_headers" yaml:"forwarded_client_ip_headers"` - forwardedClientIPSettingsLive atomic.Pointer[ForwardedClientIPSettings] `mapstructure:"-" json:"-" yaml:"-"` + TrustForwardedIPForAPIKeyACL bool `mapstructure:"trust_forwarded_ip_for_api_key_acl"` + ForwardedClientIPHeaders []string `mapstructure:"forwarded_client_ip_headers" json:"forwarded_client_ip_headers" yaml:"forwarded_client_ip_headers"` + forwardedClientIPSettingsLive *atomic.Pointer[ForwardedClientIPSettings] `mapstructure:"-" json:"-" yaml:"-"` } func NormalizeForwardedClientIPHeaders(headers []string) ([]string, error) { @@ -723,10 +723,13 @@ func (c *Config) ForwardedClientIPSettings() ForwardedClientIPSettings { if c == nil { return ForwardedClientIPSettings{Headers: []string{}} } - if snapshot := c.Security.forwardedClientIPSettingsLive.Load(); snapshot != nil { - return ForwardedClientIPSettings{ - TrustForwardedIP: snapshot.TrustForwardedIP, - Headers: cloneForwardedClientIPHeaders(snapshot.Headers), + live := c.Security.forwardedClientIPSettingsLive + if live != nil { + if snapshot := live.Load(); snapshot != nil { + return ForwardedClientIPSettings{ + TrustForwardedIP: snapshot.TrustForwardedIP, + Headers: cloneForwardedClientIPHeaders(snapshot.Headers), + } } } return ForwardedClientIPSettings{ @@ -750,6 +753,9 @@ func (c *Config) SetForwardedClientIPSettings(enabled bool, headers []string) { return } headers = cloneForwardedClientIPHeaders(headers) + if c.Security.forwardedClientIPSettingsLive == nil { + c.Security.forwardedClientIPSettingsLive = &atomic.Pointer[ForwardedClientIPSettings]{} + } c.Security.forwardedClientIPSettingsLive.Store(&ForwardedClientIPSettings{ TrustForwardedIP: enabled, Headers: headers, From a2acbf553b9e5e51bd91313e7208a8b1f89082ff Mon Sep 17 00:00:00 2001 From: wucm667 Date: Mon, 20 Jul 2026 02:20:58 +0800 Subject: [PATCH 56/59] fix(handler): avoid cache billing on same-account retry --- backend/internal/handler/failover_loop.go | 11 ++--- .../internal/handler/failover_loop_test.go | 42 ++++++++++++++++++- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/backend/internal/handler/failover_loop.go b/backend/internal/handler/failover_loop.go index 7ce1d03a8c..6b4858c528 100644 --- a/backend/internal/handler/failover_loop.go +++ b/backend/internal/handler/failover_loop.go @@ -83,8 +83,9 @@ func (s *FailoverState) HandleFailoverError( return FailoverExhausted } - // 缓存计费判断 - if needForceCacheBilling(s.hasBoundSession, failoverErr) { + // 同账号重试不算切换账号，粘性会话仅在实际切换时强制缓存计费。 + sameAccountRetry := failoverErr.RetryableOnSameAccount && s.SameAccountRetryCount[accountID] < retryLimit + if needForceCacheBilling(s.hasBoundSession, failoverErr, sameAccountRetry) { s.ForceCacheBilling = true } @@ -174,9 +175,9 @@ func (s *FailoverState) HandleSelectionExhausted(ctx context.Context) FailoverAc } // needForceCacheBilling 判断 failover 时是否需要强制缓存计费。 -// 粘性会话切换账号、或上游明确标记时，将 input_tokens 转为 cache_read 计费。 -func needForceCacheBilling(hasBoundSession bool, failoverErr *service.UpstreamFailoverError) bool { - return hasBoundSession || (failoverErr != nil && failoverErr.ForceCacheBilling) +// 粘性会话实际切换账号、或上游明确标记时，将 input_tokens 转为 cache_read 计费。 +func needForceCacheBilling(hasBoundSession bool, failoverErr *service.UpstreamFailoverError, sameAccountRetry bool) bool { + return (hasBoundSession && !sameAccountRetry) || (failoverErr != nil && failoverErr.ForceCacheBilling) } // failoverClientGone 判断下游客户端是否已断开（请求 context 已取消）。 diff --git a/backend/internal/handler/failover_loop_test.go b/backend/internal/handler/failover_loop_test.go index 8946eb5236..34644711eb 100644 --- a/backend/internal/handler/failover_loop_test.go +++ b/backend/internal/handler/failover_loop_test.go @@ -271,7 +271,7 @@ func TestHandleFailoverError_BasicSwitch(t *testing.T) { // --------------------------------------------------------------------------- func TestHandleFailoverError_CacheBilling(t *testing.T) { - t.Run("hasBoundSession为true时设置ForceCacheBilling", func(t *testing.T) { + t.Run("hasBoundSession为true且实际切换时设置ForceCacheBilling", func(t *testing.T) { mock := &mockTempUnscheduler{} fs := NewFailoverState(3, true) // hasBoundSession=true err := newTestFailoverErr(500, false, false) @@ -280,6 +280,32 @@ func TestHandleFailoverError_CacheBilling(t *testing.T) { require.True(t, fs.ForceCacheBilling) }) + t.Run("同账号重试时仅凭hasBoundSession不设置ForceCacheBilling", func(t *testing.T) { + mock := &mockTempUnscheduler{} + fs := NewFailoverState(3, true) + err := newTestFailoverErr(400, true, false) + + fs.HandleFailoverError(context.Background(), mock, 100, "openai", maxSameAccountRetries, err) + + require.False(t, fs.ForceCacheBilling) + require.Zero(t, fs.SwitchCount) + }) + + t.Run("同账号重试耗尽并实际切换时设置ForceCacheBilling", func(t *testing.T) { + mock := &mockTempUnscheduler{} + fs := NewFailoverState(3, true) + err := newTestFailoverErr(400, true, false) + + for i := 0; i < maxSameAccountRetries; i++ { + fs.HandleFailoverError(context.Background(), mock, 100, "openai", maxSameAccountRetries, err) + require.False(t, fs.ForceCacheBilling) + } + fs.HandleFailoverError(context.Background(), mock, 100, "openai", maxSameAccountRetries, err) + + require.True(t, fs.ForceCacheBilling) + require.Equal(t, 1, fs.SwitchCount) + }) + t.Run("failoverErr.ForceCacheBilling为true时设置", func(t *testing.T) { mock := &mockTempUnscheduler{} fs := NewFailoverState(3, false) @@ -289,6 +315,17 @@ func TestHandleFailoverError_CacheBilling(t *testing.T) { require.True(t, fs.ForceCacheBilling) }) + t.Run("同账号重试保留显式ForceCacheBilling", func(t *testing.T) { + mock := &mockTempUnscheduler{} + fs := NewFailoverState(3, true) + err := newTestFailoverErr(400, true, true) + + fs.HandleFailoverError(context.Background(), mock, 100, "openai", maxSameAccountRetries, err) + + require.True(t, fs.ForceCacheBilling) + require.Zero(t, fs.SwitchCount) + }) + t.Run("两者均为false时不设置", func(t *testing.T) { mock := &mockTempUnscheduler{} fs := NewFailoverState(3, false) @@ -633,13 +670,14 @@ func TestHandleFailoverError_IntegrationScenario(t *testing.T) { for i := 0; i < maxSameAccountRetries; i++ { action := fs.HandleFailoverError(context.Background(), mock, 100, "openai", maxSameAccountRetries, retryErr) require.Equal(t, FailoverContinue, action) + require.False(t, fs.ForceCacheBilling, "同账号重试期间不应仅因绑定会话强制缓存计费") } - require.True(t, fs.ForceCacheBilling, "hasBoundSession=true 应设置 ForceCacheBilling") // 2. 账号 100 超过重试上限 → TempUnschedule + 切换 action := fs.HandleFailoverError(context.Background(), mock, 100, "openai", maxSameAccountRetries, retryErr) require.Equal(t, FailoverContinue, action) require.Equal(t, 1, fs.SwitchCount) + require.True(t, fs.ForceCacheBilling, "实际切换账号时应设置 ForceCacheBilling") require.Len(t, mock.calls, 1) // 3. 账号 200 遇到不可重试错误 → 直接切换 From 30202f82663627eb1a8b661be25ee3eb7c519cb8 Mon Sep 17 00:00:00 2001 From: shaw Date: Mon, 20 Jul 2026 08:54:57 +0800 Subject: [PATCH 57/59] chore: update sponsors --- README.md | 4 ++-- README_CN.md | 4 ++-- README_JA.md | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f48e09087b..0c18411c59 100644 --- a/README.md +++ b/README.md @@ -77,8 +77,8 @@ Please read the following carefully before using this project:
pptokenThanks to PPToken.org for sponsoring this project! PPToken.org specializes in GPT model API relay services, supporting Codex, Claude Code, OpenAI-compatible clients, and Gemini CLI integration. Top-ups are 1:1 (¥1 = $1 credit); GPT models start at 0.16x rate multiplier, with overall cost at roughly 2.2% of official pricing and first-token latency around 1 second — ideal for developers seeking low-cost, high-speed access to GPT model capabilities. Technical support: 24/7 real human responses (no bots), @tech in the group chat and get a reply within 10 minutes. Sponsor benefit: the first 200 users who register via the exclusive registration link and enter promo code `SUB2API` can claim free Codex / Claude Code trial credits — no minimum spend, no card required. +pptokenThanks to PPToken.cc for sponsoring this project! PPToken.cc specializes in GPT model API relay services, supporting Codex, Claude Code, OpenAI-compatible clients, and Gemini CLI integration. Top-ups are 1:1 (¥1 = $1 credit); GPT models start at 0.16x rate multiplier, with overall cost at roughly 2.2% of official pricing and first-token latency around 1 second — ideal for developers seeking low-cost, high-speed access to GPT model capabilities. Technical support: 24/7 real human responses (no bots), @tech in the group chat and get a reply within 10 minutes. Sponsor benefit: the first 200 users who register via the exclusive registration link and enter promo code `SUB2API` can claim free Codex / Claude Code trial credits — no minimum spend, no card required.
pptoken感谢 PPToken.org 赞助本项目！ PPToken.org 主打 GPT 系列模型 API 中转服务，支持 Codex、Claude Code、OpenAI 兼容客户端及 Gemini CLI 等工具接入。充值 1:1，1 元=1 美元额度；GPT 模型最低 0.16 倍倍率，综合成本约为官方价格的 0.22 折，最快首字 Token 约 1 秒，适合开发者低成本、高响应速度接入 GPT 模型能力。技术支持： 7×24 小时真人响应（不是机器人），群内@技术，10 分钟内有回复 。赞助商福利：前 200 名用户通过 [专属注册链接] 注册，输入优惠码 `SUB2API`，可领取 Codex / Claude Code 免费试用额度，无门槛、不绑卡。 +pptoken感谢 PPToken.cc 赞助本项目！ PPToken.cc 主打 GPT 系列模型 API 中转服务，支持 Codex、Claude Code、OpenAI 兼容客户端及 Gemini CLI 等工具接入。充值 1:1，1 元=1 美元额度；GPT 模型最低 0.16 倍倍率，综合成本约为官方价格的 0.22 折，最快首字 Token 约 1 秒，适合开发者低成本、高响应速度接入 GPT 模型能力。技术支持： 7×24 小时真人响应（不是机器人），群内@技术，10 分钟内有回复 。赞助商福利：前 200 名用户通过 [专属注册链接] 注册，输入优惠码 `SUB2API`，可领取 Codex / Claude Code 免费试用额度，无门槛、不绑卡。
pptokenPPToken.org のご支援に感謝します！PPToken.org は GPT シリーズモデルの API 中継サービスを専門としており、Codex、Claude Code、OpenAI 互換クライアント、Gemini CLI などのツール接続をサポートしています。チャージは 1:1（1元＝1ドル分のクレジット）、GPT モデルは最低 0.16 倍のレート倍率で、総合コストは公式価格の約 2.2% 、最速ファーストトークンは約1秒 — 開発者が低コスト・高速レスポンスで GPT モデル機能にアクセスするのに最適です。テクニカルサポート：24時間365日リアルな人間が対応（ボットではありません）、グループ内で @技術 すれば 10 分以内に返信。スポンサー特典：先着 200 名のユーザーが専用登録リンクから登録し、プロモコード `SUB2API` を入力すると、Codex / Claude Code の無料トライアルクレジットを獲得できます — 最低利用額なし、カード登録不要。 +pptokenPPToken.cc のご支援に感謝します！PPToken.cc は GPT シリーズモデルの API 中継サービスを専門としており、Codex、Claude Code、OpenAI 互換クライアント、Gemini CLI などのツール接続をサポートしています。チャージは 1:1（1元＝1ドル分のクレジット）、GPT モデルは最低 0.16 倍のレート倍率で、総合コストは公式価格の約 2.2% 、最速ファーストトークンは約1秒 — 開発者が低コスト・高速レスポンスで GPT モデル機能にアクセスするのに最適です。テクニカルサポート：24時間365日リアルな人間が対応（ボットではありません）、グループ内で @技術 すれば 10 分以内に返信。スポンサー特典：先着 200 名のユーザーが専用登録リンクから登録し、プロモコード `SUB2API` を入力すると、Codex / Claude Code の無料トライアルクレジットを獲得できます — 最低利用額なし、カード登録不要。