diff --git a/backend/internal/service/account_scheduling_threshold_eval.go b/backend/internal/service/account_scheduling_threshold_eval.go index 345f997a15..0e1a40934c 100644 --- a/backend/internal/service/account_scheduling_threshold_eval.go +++ b/backend/internal/service/account_scheduling_threshold_eval.go @@ -271,13 +271,19 @@ func openAIThresholdCandidate(extra map[string]any, window string, now time.Time if !ok { return nil } - if openAIQuotaWindowReset(extra, window, now) || openAICodexSnapshotStaleForPause(extra, now) { + if openAIQuotaWindowReset(extra, window, now) || (openAICodexSnapshotStaleForPause(extra, now) && !openAIQuotaWindowResetPending(extra, window, now)) { return nil } + until := parseSchedulingResetAt(extra[resetAtKey]) + if until == nil { + if resetAt, ok := openAICodexWindowResetAt(extra, window); ok { + until = &resetAt + } + } return &accountSchedulingThresholdCandidate{ window: window, usedPercent: schedulingPercentValue(usedPercent), - until: parseSchedulingResetAt(extra[resetAtKey]), + until: until, } } diff --git a/backend/internal/service/account_scheduling_threshold_eval_test.go b/backend/internal/service/account_scheduling_threshold_eval_test.go index 5e7e9fa77e..5e39de87de 100644 --- a/backend/internal/service/account_scheduling_threshold_eval_test.go +++ b/backend/internal/service/account_scheduling_threshold_eval_test.go @@ -172,7 +172,7 @@ func TestEvaluateAccountSchedulingThreshold_OpenAISkipsStaleSnapshot(t *testing. Extra: map[string]any{ "codex_usage_updated_at": now.Add(-2 * time.Hour).Format(time.RFC3339), "codex_5h_used_percent": 100.0, - "codex_5h_reset_at": now.Add(3 * time.Hour).Format(time.RFC3339), + "codex_5h_reset_at": "invalid", }, } @@ -181,6 +181,63 @@ func TestEvaluateAccountSchedulingThreshold_OpenAISkipsStaleSnapshot(t *testing. require.False(t, decision.ShouldPause) } +func TestOpenAIThresholdCandidate_StaleSnapshotFutureReset(t *testing.T) { + now := time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC) + for _, tc := range []struct { + name string + reset map[string]any + want bool + }{ + {"absolute future", map[string]any{"codex_5h_reset_at": now.Add(time.Hour).Format(time.RFC3339)}, true}, + {"relative future", map[string]any{"codex_5h_reset_after_seconds": 4 * 3600}, true}, + {"missing", nil, false}, + {"invalid", map[string]any{"codex_5h_reset_at": "invalid"}, false}, + {"past", map[string]any{"codex_5h_reset_at": now.Add(-time.Minute).Format(time.RFC3339)}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + extra := map[string]any{"codex_usage_updated_at": now.Add(-3 * time.Hour).Format(time.RFC3339), "codex_5h_used_percent": 99.0} + for key, value := range tc.reset { + extra[key] = value + } + candidate := openAIThresholdCandidate(extra, "5h", now) + require.Equal(t, tc.want, candidate != nil) + if tc.want { + decision := EvaluateAccountSchedulingThreshold(&Account{Platform: PlatformOpenAI, Extra: extra}, map[string]int{PlatformOpenAI: 95}, now) + require.True(t, decision.ShouldPause) + require.NotNil(t, decision.Until) + require.True(t, now.Before(*decision.Until)) + } + }) + } +} + +func TestResolveOpenAIQuotaUtilization_StaleSnapshotFutureReset(t *testing.T) { + now := time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC) + for _, tc := range []struct { + name string + reset map[string]any + want bool + }{ + {"absolute future", map[string]any{"codex_5h_reset_at": now.Add(time.Hour).Format(time.RFC3339)}, true}, + {"relative future", map[string]any{"codex_5h_reset_after_seconds": 4 * 3600}, true}, + {"missing", nil, false}, + {"invalid", map[string]any{"codex_5h_reset_at": "invalid"}, false}, + {"past", map[string]any{"codex_5h_reset_at": now.Add(-time.Minute).Format(time.RFC3339)}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + extra := map[string]any{"codex_usage_updated_at": now.Add(-3 * time.Hour).Format(time.RFC3339), "codex_5h_used_percent": 99.0} + for key, value := range tc.reset { + extra[key] = value + } + utilization, ok := resolveOpenAIQuotaUtilization(extra, "5h", now) + require.Equal(t, tc.want, ok) + if ok { + require.Equal(t, 0.99, utilization) + } + }) + } +} + func TestEvaluateAccountSchedulingThreshold_OpenAISkipsResetWindow(t *testing.T) { t.Parallel() diff --git a/backend/internal/service/openai_account_scheduler_test.go b/backend/internal/service/openai_account_scheduler_test.go index 081cf30f9a..70cf79d7e3 100644 --- a/backend/internal/service/openai_account_scheduler_test.go +++ b/backend/internal/service/openai_account_scheduler_test.go @@ -1906,7 +1906,7 @@ func TestOpenAIGatewayService_SelectAccountForModelWithExclusions_FreshUsageWind // inversion) gets excluded from scheduling, and a paused account never receives traffic to // refresh its snapshot. When the snapshot is stale (codex_usage_updated_at older than the // staleness bound) the account must be allowed a request so it can self-heal from the real -// response headers — independent of the window's reset time. +// response headers when no valid future window reset is known. func TestOpenAIGatewayService_SelectAccountForModelWithExclusions_StaleUsageSnapshotSkipsPause_Issue2994(t *testing.T) { ctx := context.Background() primary := Account{ @@ -1920,8 +1920,7 @@ func TestOpenAIGatewayService_SelectAccountForModelWithExclusions_StaleUsageSnap Extra: map[string]any{ "codex_5h_used_percent": 99.0, "auto_pause_5h_threshold": 0.95, - // Window has NOT reset yet, so the reset guard stays inactive. - "codex_5h_reset_at": time.Now().Add(time.Hour).Format(time.RFC3339), + // No reset time is known, so stale snapshot self-heal applies. // Snapshot is stale: older than openAICodexAutoPauseStaleAfter (2h). "codex_usage_updated_at": time.Now().Add(-3 * time.Hour).Format(time.RFC3339), }, diff --git a/backend/internal/service/openai_gateway_scheduling.go b/backend/internal/service/openai_gateway_scheduling.go index 2f9a750caa..5f5bf84e25 100644 --- a/backend/internal/service/openai_gateway_scheduling.go +++ b/backend/internal/service/openai_gateway_scheduling.go @@ -662,8 +662,8 @@ func resolveAccountExtraNumber(extra map[string]any, keys ...string) (float64, b // resolveOpenAIQuotaUtilization returns the current utilization ratio (0..1) for the // given Codex usage window. ok=false means there is no usable signal to pause on: -// either no snapshot exists, or the window has already rolled over so the cached -// percentage is stale. The stale guard matters because a paused account stops +// either no snapshot exists, the window has rolled over, or an old snapshot has +// no known future reset. The stale guard matters because a paused account stops // receiving requests, so its snapshot is never refreshed from upstream headers — // without this check an old used_percent would keep the account paused forever even // after the real window reset. @@ -675,14 +675,19 @@ func resolveOpenAIQuotaUtilization(extra map[string]any, window string, now time if openAIQuotaWindowReset(extra, window, now) { return 0, false } - // 快照过于陈旧（账号长期未收到流量刷新）时，不再据此暂停。放行后下一次响应头 + // 快照过于陈旧且没有明确的未来重置时间时，不再据此暂停。放行后下一次响应头 // 会刷新快照实现自愈，避免账号在错误/过期的 used% 上被永久跳过（issue #2994）。 - if openAICodexSnapshotStaleForPause(extra, now) { + if openAICodexSnapshotStaleForPause(extra, now) && !openAIQuotaWindowResetPending(extra, window, now) { return 0, false } return usedPercent / 100, true } +func openAIQuotaWindowResetPending(extra map[string]any, window string, now time.Time) bool { + resetAt, ok := openAICodexWindowResetAt(extra, window) + return ok && now.Before(resetAt) +} + // openAICodexSnapshotStaleForPause reports whether the Codex usage snapshot is stale // enough that it should no longer keep an account auto-paused. It anchors on // codex_usage_updated_at (always written by buildCodexUsageExtraUpdates). A missing or