From 56f3d3c9b0ab23a68d648f97f2b011707b9f016c Mon Sep 17 00:00:00 2001 From: shaw Date: Sat, 1 Aug 2026 21:53:55 +0800 Subject: [PATCH] =?UTF-8?q?fix(billing-probe):=20=E6=94=B6=E5=8F=A3?= =?UTF-8?q?=E6=8E=A2=E6=B5=8B=E8=B5=84=E6=A0=BC=E6=94=BE=E5=AE=BD=E5=90=8E?= =?UTF-8?q?=E7=9A=84=E6=8A=91=E5=88=B6=E6=B8=85=E5=8D=95=E4=B8=8E=E8=B0=83?= =?UTF-8?q?=E5=BA=A6=E4=BF=A1=E4=BB=BB=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #4896 把上游计费探测从 OpenAI API-key 放宽到全部 API-key 账号后， 遗留了四处需要收口的问题： - M1 官方域抑制清单补 ollama.com。Ollama Cloud 是本仓一等支持配置 （platform openai/anthropic + type apikey + base_url https://ollama.com/v1），放宽后 anthropic 侧这类账号会每个探测周期 拿 Ollama Key 请求 ollama.com/v1/sub2api/billing 并恒定落空，正是该 守卫注释声明要防的行为。后缀匹配已覆盖 www.ollama.com 等子域， notollama.com 等形似域不受影响。 - M2 legacy 低倍率优先排序补平台门控。newOpenAILegacyUpstreamRateOrder 遍历全部候选且无平台门控，与 openAIUpstreamCostFactors 的门控不对称； 放宽后 grok 账号的上游自报倍率开始影响 legacy 调度排序，而实际结算走 本地倍率，中转方自报低价即可吸流量。现补上同一道门控，使调度侧信任面 回到 PR 前状态（探测资格的放宽保持不变）。 - L1 修正 IsUpstreamBillingProbeIdentity 注释。原注释称类型限制的依据是 "OAuth/Bedrock 没有静态 API key"，但 AccountTypeUpstream（antigravity 中转账号）同样是 base_url + 静态 api_key 却也被排除。仅改注释如实说明 取舍，不改行为。 - L2 批量探测空选文案去掉 OpenAI 限定（en/zh 成对）。 - L3 unsupported 状态改用加长退避（interval 的 8 倍，仍按 24h 封顶）。 放宽后大量官方域账号会落 unsupported 并按常规 interval 重排，占满每周期 20 个名额，把真正接入 sub2api 的中转账号挤到后面。封顶保证上游后来接入时 最迟一天内会被重新发现；Retry-After 更长时原样保留不被缩短；手动探测不受 退避影响。 测试：ollama.com 官方域行为级与 host 匹配矩阵用例、legacy 排序平台门控 （含混合候选集）用例、unsupported 退避上下界与 runner 跳过/手动探测放行 用例；同步更新既有 unsupported 的 next_probe_at 断言。 --- .../service/openai_account_scheduler.go | 9 +++ ...ai_account_scheduler_upstream_cost_test.go | 40 +++++++++++ .../service/upstream_billing_probe.go | 37 +++++++++- ...stream_billing_probe_multiplatform_test.go | 13 ++++ .../service/upstream_billing_probe_test.go | 72 +++++++++++++++++-- .../src/i18n/locales/en/admin/accounts.ts | 2 +- .../src/i18n/locales/zh/admin/accounts.ts | 2 +- 7 files changed, 166 insertions(+), 9 deletions(-) diff --git a/backend/internal/service/openai_account_scheduler.go b/backend/internal/service/openai_account_scheduler.go index ceafdda40a..c82e845a86 100644 --- a/backend/internal/service/openai_account_scheduler.go +++ b/backend/internal/service/openai_account_scheduler.go @@ -2653,6 +2653,15 @@ func newOpenAILegacyUpstreamRateOrder(accounts []*Account, now time.Time, oauthS var first float64 distinct := false for _, account := range accounts { + if account == nil { + continue + } + // 与 openAIUpstreamCostFactors 使用同一道平台门控：只有 OpenAI 平台账号 + // 的倍率参与 legacy 低倍率优先排序。上游自报倍率来自中转方，不能让它对 + // 其他平台的调度产生影响——否则自报低价即可吸走流量，而实际结算走本地倍率。 + if !account.IsOpenAIApiKey() && !account.IsOpenAIOAuth() { + continue + } rate, ok := openAISchedulingRate(account, now, oauthSchedulingRateMultiplier) if !ok { continue diff --git a/backend/internal/service/openai_account_scheduler_upstream_cost_test.go b/backend/internal/service/openai_account_scheduler_upstream_cost_test.go index 6c32db4ee8..1288107573 100644 --- a/backend/internal/service/openai_account_scheduler_upstream_cost_test.go +++ b/backend/internal/service/openai_account_scheduler_upstream_cost_test.go @@ -491,6 +491,46 @@ func TestOpenAILegacyUpstreamRateOrderRequiresComparableRates(t *testing.T) { require.Negative(t, distinct.compare(&Account{ID: 2}, &Account{ID: 3})) } +// 探测资格已放宽到全部 API-key 平台，但调度侧的信任面没有跟着扩大： +// 只有 OpenAI 平台账号的上游自报倍率参与 legacy 低倍率优先排序， +// 否则中转方自报低价即可吸走流量，而实际结算走本地倍率。 +// 本用例钉死 newOpenAILegacyUpstreamRateOrder 与 openAIUpstreamCostFactors +// 使用同一道平台门控。 +func TestOpenAILegacyUpstreamRateOrderIgnoresNonOpenAIPlatforms(t *testing.T) { + now := time.Now() + nonOpenAI := func(id int64, platform string, rate float64) *Account { + account := upstreamCostTestAccount(id, UpstreamBillingProbeStatusOK, rate, now.Add(-time.Minute), 30*time.Minute) + account.Platform = platform + return account + } + grokCheap := nonOpenAI(1, PlatformGrok, 0.01) + anthropicExpensive := nonOpenAI(2, PlatformAnthropic, 0.9) + + order := newOpenAILegacyUpstreamRateOrder([]*Account{grokCheap, anthropicExpensive, nil}, now, defaultOpenAIOAuthSchedulingRateMultiplier) + require.False(t, order.enabled) + require.Empty(t, order.rates) + require.Zero(t, order.compare(grokCheap, anthropicExpensive)) + + factors := openAIUpstreamCostFactors([]*Account{grokCheap, anthropicExpensive}, now, defaultOpenAIOAuthSchedulingRateMultiplier) + require.Equal(t, openAIUpstreamCostNeutralFactor, factors[grokCheap.ID]) + require.Equal(t, openAIUpstreamCostNeutralFactor, factors[anthropicExpensive.ID]) + + // 混合候选集里，非 OpenAI 账号既不进 rates 也不影响 OpenAI 账号之间的排序。 + openAICheap := upstreamCostTestAccount(3, UpstreamBillingProbeStatusOK, 0.02, now.Add(-time.Minute), 30*time.Minute) + openAIExpensive := upstreamCostTestAccount(4, UpstreamBillingProbeStatusOK, 0.12, now.Add(-time.Minute), 30*time.Minute) + mixed := newOpenAILegacyUpstreamRateOrder( + []*Account{grokCheap, openAICheap, anthropicExpensive, openAIExpensive}, + now, defaultOpenAIOAuthSchedulingRateMultiplier, + ) + require.True(t, mixed.enabled) + require.Len(t, mixed.rates, 2) + require.NotContains(t, mixed.rates, grokCheap.ID) + require.NotContains(t, mixed.rates, anthropicExpensive.ID) + require.Negative(t, mixed.compare(openAICheap, openAIExpensive)) + // 自报 0.01 的 grok 账号没有已知倍率，排在有倍率的 OpenAI 账号之后。 + require.Positive(t, mixed.compare(grokCheap, openAIExpensive)) +} + func TestOpenAISchedulingRatePlacesOAuthAtConfiguredReference(t *testing.T) { now := time.Now() cheap := upstreamCostTestAccount(1, UpstreamBillingProbeStatusOK, 0.02, now.Add(-time.Minute), 30*time.Minute) diff --git a/backend/internal/service/upstream_billing_probe.go b/backend/internal/service/upstream_billing_probe.go index 9ddb55f3ed..7f105445e2 100644 --- a/backend/internal/service/upstream_billing_probe.go +++ b/backend/internal/service/upstream_billing_probe.go @@ -40,6 +40,10 @@ const ( upstreamBillingProbeMaxPerCycle = 20 upstreamBillingProbeConcurrency = 4 upstreamBillingProbeMaxDelay = 24 * time.Hour + // unsupported 账号的重探间隔倍数：上游不是 sub2api 中转就不会突然长出 + // /v1/sub2api/billing，按常规 interval 重排只会持续占满每周期 + // upstreamBillingProbeMaxPerCycle 个名额。 + upstreamBillingProbeUnsupportedDelayFactor = 8 upstreamBillingProbeLeaderLockKey = "upstream:billing:probe:leader" upstreamBillingProbeLeaderLockTTL = 2 * time.Minute ) @@ -661,13 +665,15 @@ func (s *UpstreamBillingProbeService) persistProbeFailure( failureCount = previous.FailureCount + 1 } status := UpstreamBillingProbeStatusFailed + delay := nextProbeDelay(intervalMinutes, retryAfterDuration) if reason == "unsupported" { status = UpstreamBillingProbeStatusUnsupported + delay = unsupportedProbeDelay(intervalMinutes, retryAfterDuration) } snapshot := &UpstreamBillingProbeSnapshot{ Status: status, LastAttemptAt: now, - NextProbeAt: now.Add(nextProbeDelay(intervalMinutes, retryAfterDuration)), + NextProbeAt: now.Add(delay), FailureCount: failureCount, HTTPStatus: statusCode, LastError: reason, @@ -863,8 +869,12 @@ func decodeUpstreamBillingProbeSnapshot(extra map[string]any) *UpstreamBillingPr // sub2api-compatible upstream answers it regardless of the account platform, // so eligibility is not restricted to OpenAI. // Non-sub2api upstreams return 404 and the snapshot records "unsupported". -// OAuth/Bedrock credentials carry no static API key to present, hence the -// type restriction. +// Only AccountTypeAPIKey is in scope. OAuth/Bedrock hold no static API key to +// present at all; AccountTypeUpstream (antigravity relay accounts) does carry +// a base_url plus a static api_key, but it is deliberately left out of the +// current supported set. New antigravity relay accounts are created with +// type=apikey by the admin form, so only pre-existing type=upstream rows +// cannot turn the probe on. func IsUpstreamBillingProbeIdentity(platform, accountType string) bool { return platform != "" && accountType == AccountTypeAPIKey } @@ -883,12 +893,16 @@ func isUpstreamBillingProbeAccount(account *Account) bool { // relays (the only targets that can answer /v1/sub2api/billing) always do // probe. OpenAI-platform accounts never reach this check: they keep the // upstream-official behavior of probing api.openai.com. +// ollama.com is a first-class configuration here (Ollama Cloud accounts are +// platform openai/anthropic with base_url https://ollama.com/v1), and it is +// an official provider API just like the rest, so it belongs on this list. var upstreamBillingProbeOfficialAPIDomains = []string{ "anthropic.com", "googleapis.com", "x.ai", "grok.com", "openai.com", + "ollama.com", } func upstreamBillingProbeTargetIsOfficialAPI(baseURL string) bool { @@ -953,6 +967,23 @@ func nextProbeDelay(intervalMinutes int, retryAfterDuration time.Duration) time. return interval } +// unsupportedProbeDelay 拉长 unsupported 账号的重探间隔，让无效候选自然退出 +// 热队列，不再和真正接入 sub2api 的中转账号抢每周期的探测名额。 +// 仍按 upstreamBillingProbeMaxDelay 封顶，保证上游后来接入 sub2api 时最迟一天 +// 内会被重新发现；base 本身已达上限（例如 Retry-After 明确要求更久）时原样返回， +// 不缩短上游指令。 +func unsupportedProbeDelay(intervalMinutes int, retryAfterDuration time.Duration) time.Duration { + base := nextProbeDelay(intervalMinutes, retryAfterDuration) + if base >= upstreamBillingProbeMaxDelay { + return base + } + stretched := base * upstreamBillingProbeUnsupportedDelayFactor + if stretched > upstreamBillingProbeMaxDelay { + return upstreamBillingProbeMaxDelay + } + return stretched +} + func retryAfter(header http.Header, now time.Time) time.Duration { value := strings.TrimSpace(header.Get("Retry-After")) if value == "" { diff --git a/backend/internal/service/upstream_billing_probe_multiplatform_test.go b/backend/internal/service/upstream_billing_probe_multiplatform_test.go index c95e159ed8..a2b03a96fc 100644 --- a/backend/internal/service/upstream_billing_probe_multiplatform_test.go +++ b/backend/internal/service/upstream_billing_probe_multiplatform_test.go @@ -118,6 +118,11 @@ func TestUpstreamBillingProbeOfficialAPIBaseURLIsUnsupportedWithoutRequest(t *te // 跨平台官方域同样必无该端点，一并拦截。 {PlatformAnthropic, "https://api.x.ai/v1"}, {PlatformGrok, "https://api.openai.com"}, + // Ollama Cloud 是本仓一等支持配置（platform openai/anthropic + + // base_url https://ollama.com/v1），同为官方 API，不能拿 Key 去空探。 + {PlatformAnthropic, "https://ollama.com/v1"}, + {PlatformAnthropic, "https://ollama.com"}, + {PlatformAnthropic, "https://www.ollama.com/v1"}, } for i, tc := range cases { account := &Account{ @@ -149,11 +154,19 @@ func TestUpstreamBillingProbeOfficialAPIHostMatchingIsNormalized(t *testing.T) { require.True(t, upstreamBillingProbeTargetIsOfficialAPI("https://x.ai")) // openai 官方域在全集内；openai 平台账号不经过本判定（行为级测试钉死照探）。 require.True(t, upstreamBillingProbeTargetIsOfficialAPI("https://api.openai.com")) + // Ollama Cloud 官方域及其子域（Ollama Cloud 账号的 base_url 允许带 www.）。 + require.True(t, upstreamBillingProbeTargetIsOfficialAPI("https://ollama.com/v1")) + require.True(t, upstreamBillingProbeTargetIsOfficialAPI("https://ollama.com:443/v1")) + require.True(t, upstreamBillingProbeTargetIsOfficialAPI("https://www.ollama.com/v1")) + require.True(t, upstreamBillingProbeTargetIsOfficialAPI("HTTPS://OLLAMA.COM./v1")) // 相似但不同的注册域不拦：中转完全可能叫 *-x.ai 之外的任何名字。 require.False(t, upstreamBillingProbeTargetIsOfficialAPI("https://relay.example/v1")) require.False(t, upstreamBillingProbeTargetIsOfficialAPI("https://notx.ai")) require.False(t, upstreamBillingProbeTargetIsOfficialAPI("https://anthropic.com.evil.example")) require.False(t, upstreamBillingProbeTargetIsOfficialAPI("https://api.relay-station.example")) + require.False(t, upstreamBillingProbeTargetIsOfficialAPI("https://notollama.com/v1")) + require.False(t, upstreamBillingProbeTargetIsOfficialAPI("https://ollama.com.evil.example/v1")) + require.False(t, upstreamBillingProbeTargetIsOfficialAPI("https://ollama.example/v1")) } // OpenAI 语义保持不变：无自定义 base 时仍探官方域，且沿用 openai 传输画像。 diff --git a/backend/internal/service/upstream_billing_probe_test.go b/backend/internal/service/upstream_billing_probe_test.go index eecd866a90..edd297af96 100644 --- a/backend/internal/service/upstream_billing_probe_test.go +++ b/backend/internal/service/upstream_billing_probe_test.go @@ -499,6 +499,69 @@ func TestUpstreamBillingProbeRetryAfterIsNotShortened(t *testing.T) { require.Equal(t, 48*time.Hour, delay) } +// unsupported 的重探间隔明显长于普通失败，但始终有上界：上游后来接入 sub2api +// 时最迟一天内会被重新发现，且不会缩短上游 Retry-After 指令。 +func TestUpstreamBillingProbeUnsupportedDelayIsStretchedAndBounded(t *testing.T) { + // 默认 30 分钟 interval：普通失败 24~36 分钟，unsupported 为其 8 倍。 + stretched := unsupportedProbeDelay(30, 0) + require.Greater(t, stretched, 36*time.Minute) + require.GreaterOrEqual(t, stretched, 192*time.Minute) + require.LessOrEqual(t, stretched, 288*time.Minute) + + // 永不超过封顶值，因此 unsupported 账号不会被永久排除在重探之外。 + require.LessOrEqual(t, unsupportedProbeDelay(upstreamBillingProbeMaxIntervalMinutes, 0), upstreamBillingProbeMaxDelay) + require.Positive(t, unsupportedProbeDelay(upstreamBillingProbeMinIntervalMinutes, 0)) + + // Retry-After 更长时原样保留，不被封顶缩短；更短时至少不早于该指令。 + require.Equal(t, 48*time.Hour, unsupportedProbeDelay(30, 48*time.Hour)) + require.GreaterOrEqual(t, unsupportedProbeDelay(30, time.Hour), time.Hour) +} + +// 加长退避只把 unsupported 账号移出周期性热队列，手动探测不受影响。 +func TestUpstreamBillingProbeUnsupportedBackoffDefersRunnerButNotManualProbe(t *testing.T) { + // Ollama Cloud 形态：官方域，不发请求直接落 unsupported。 + account := &Account{ + ID: 31, + Platform: PlatformAnthropic, + Type: AccountTypeAPIKey, + Status: StatusActive, + Concurrency: 1, + Credentials: map[string]any{"api_key": "sk-ollama", "base_url": "https://ollama.com/v1"}, + Extra: map[string]any{UpstreamBillingProbeEnabledExtraKey: true}, + } + repo := &upstreamBillingProbeAccountRepo{accounts: map[int64]*Account{account.ID: account}} + upstream := &upstreamBillingProbeHTTPStub{} + settingsRepo := &upstreamBillingProbeSettingRepo{values: map[string]string{ + SettingKeyUpstreamBillingProbeSettings: `{"enabled":true,"interval_minutes":30}`, + }} + svc := newUpstreamBillingProbeTestService(repo, upstream, settingsRepo) + start := time.Date(2026, time.July, 26, 2, 0, 0, 0, time.UTC) + now := start + svc.now = func() time.Time { return now } + + require.NoError(t, svc.RunDue(context.Background())) + first := decodeUpstreamBillingProbeSnapshot(account.Extra) + require.NotNil(t, first) + require.Equal(t, UpstreamBillingProbeStatusUnsupported, first.Status) + require.Equal(t, start, first.LastAttemptAt) + require.False(t, first.NextProbeAt.Before(start.Add(192*time.Minute))) + require.Zero(t, upstream.calls.Load()) + + // 一个普通失败早就该重探的时间点（远超 36 分钟），runner 仍跳过该账号。 + now = start.Add(90 * time.Minute) + require.NoError(t, svc.RunDue(context.Background())) + deferred := decodeUpstreamBillingProbeSnapshot(account.Extra) + require.NotNil(t, deferred) + require.Equal(t, first.LastAttemptAt, deferred.LastAttemptAt) + + // 手动探测无视退避窗口，管理员随时可以重试。 + manual, err := svc.ProbeAccount(context.Background(), account.ID) + require.NoError(t, err) + require.Equal(t, UpstreamBillingProbeStatusUnsupported, manual.Status) + require.Equal(t, now, manual.LastAttemptAt) + require.Equal(t, 2, manual.FailureCount) +} + func TestUpstreamBillingProbeEmptyResponseIsPersistedAsFailure(t *testing.T) { account := &Account{ ID: 21, @@ -552,14 +615,15 @@ func TestUpstreamBillingProbeUnsupportedAndAccountToggle(t *testing.T) { require.NoError(t, err) require.Equal(t, UpstreamBillingProbeStatusUnsupported, snapshot.Status) require.Equal(t, "unsupported", snapshot.LastError) - require.False(t, snapshot.NextProbeAt.Before(fixedNow.Add(24*time.Minute))) - require.False(t, snapshot.NextProbeAt.After(fixedNow.Add(36*time.Minute))) + // unsupported 走加长退避：默认 30 分钟 interval ⇒ (24~36) * 8 分钟。 + require.False(t, snapshot.NextProbeAt.Before(fixedNow.Add(192*time.Minute))) + require.False(t, snapshot.NextProbeAt.After(fixedNow.Add(288*time.Minute))) snapshot, err = svc.ProbeAccount(context.Background(), account.ID) require.NoError(t, err) require.Equal(t, 2, snapshot.FailureCount) - require.False(t, snapshot.NextProbeAt.Before(fixedNow.Add(24*time.Minute))) - require.False(t, snapshot.NextProbeAt.After(fixedNow.Add(36*time.Minute))) + require.False(t, snapshot.NextProbeAt.Before(fixedNow.Add(192*time.Minute))) + require.False(t, snapshot.NextProbeAt.After(fixedNow.Add(288*time.Minute))) invalid := &Account{ID: 20, Platform: PlatformOpenAI, Type: AccountTypeOAuth} repo.accounts[invalid.ID] = invalid diff --git a/frontend/src/i18n/locales/en/admin/accounts.ts b/frontend/src/i18n/locales/en/admin/accounts.ts index ce10004d91..9a260d6170 100644 --- a/frontend/src/i18n/locales/en/admin/accounts.ts +++ b/frontend/src/i18n/locales/en/admin/accounts.ts @@ -250,7 +250,7 @@ export default { enabled: 'On', disabled: 'Off', probeFailed: 'Failed to probe upstream rate', - noEligibleAccounts: 'Select OpenAI API key accounts', + noEligibleAccounts: 'Select API key accounts', batchLimit: 'A batch can probe at most 20 accounts', batchCompleted: 'Probed {count} account(s)', batchPartial: 'Probe partially completed: {success} succeeded, {failed} failed' diff --git a/frontend/src/i18n/locales/zh/admin/accounts.ts b/frontend/src/i18n/locales/zh/admin/accounts.ts index 578238c40c..391dc23355 100644 --- a/frontend/src/i18n/locales/zh/admin/accounts.ts +++ b/frontend/src/i18n/locales/zh/admin/accounts.ts @@ -203,7 +203,7 @@ export default { enabled: '打开', disabled: '关闭', probeFailed: '探测上游倍率失败', - noEligibleAccounts: '请选择 OpenAI API Key 账号', + noEligibleAccounts: '请选择 API Key 账号', batchLimit: '每次最多探测 20 个账号', batchCompleted: '已完成 {count} 个账号的倍率探测', batchPartial: '倍率探测部分完成：成功 {success} 个，失败 {failed} 个'