Merge pull request #2800 from wucm667/fix/scheduler-model-not-found-per-model-cooldown

fix(scheduler): 模型 404 仅冷却该账号-模型组合,不再封整个账号
This commit is contained in:
Wesley Liddick authored and GitHub committed 2026-05-27 21:01:52 +08:00
commit 61ce79533e
26 files changed
+434 -63

No files matched your search

@@ -1312,8 +1312,8 @@ func openAICompactSupportTier(account *Account) int {
// isOpenAIAccountEligibleForRequest centralises the schedulable / OpenAI / model /
// compact-support checks used during account selection.
func isOpenAIAccountEligibleForRequest(account *Account, requestedModel string, requireCompact bool) bool {
if account == nil || !account.IsSchedulable() || !account.IsOpenAI() {
func isOpenAIAccountEligibleForRequest(ctx context.Context, account *Account, requestedModel string, requireCompact bool) bool {
if account == nil || !account.IsOpenAI() || !account.IsSchedulableForModelWithContext(ctx, requestedModel) {
return false
}
if requestedModel != "" && !account.IsModelSupported(requestedModel) {
@@ -1446,7 +1446,7 @@ func (s *OpenAIGatewayService) tryStickySessionHit(ctx context.Context, groupID
// 验证账号是否可用于当前请求
// Verify account is usable for current request
if !isOpenAIAccountEligibleForRequest(account, requestedModel, false) {
if !isOpenAIAccountEligibleForRequest(ctx, account, requestedModel, false) {
return nil
}
if s.isOpenAIAccountRuntimeBlocked(account) {
@@ -1646,7 +1646,7 @@ func (s *OpenAIGatewayService) selectAccountWithLoadAwareness(ctx context.Contex
if clearSticky {
_ = s.deleteStickySessionAccountID(ctx, groupID, sessionHash)
}
if !clearSticky && isOpenAIAccountEligibleForRequest(account, requestedModel, false) {
if !clearSticky && isOpenAIAccountEligibleForRequest(ctx, account, requestedModel, false) {
account = s.recheckSelectedOpenAIAccountFromDB(ctx, account, requestedModel, requireCompact)
if account == nil {
_ = s.deleteStickySessionAccountID(ctx, groupID, sessionHash)
@@ -1924,7 +1924,7 @@ func (s *OpenAIGatewayService) resolveFreshSchedulableOpenAIAccount(ctx context.
fresh = current
}
if !isOpenAIAccountEligibleForRequest(fresh, requestedModel, requireCompact) {
if !isOpenAIAccountEligibleForRequest(ctx, fresh, requestedModel, requireCompact) {
return nil
}
if s.isOpenAIAccountRuntimeBlocked(fresh) {
@@ -1938,7 +1938,7 @@ func (s *OpenAIGatewayService) recheckSelectedOpenAIAccountFromDB(ctx context.Co
return nil
}
if s.schedulerSnapshot == nil || s.accountRepo == nil {
if !isOpenAIAccountEligibleForRequest(account, requestedModel, requireCompact) {
if !isOpenAIAccountEligibleForRequest(ctx, account, requestedModel, requireCompact) {
return nil
}
return account
@@ -1948,7 +1948,7 @@ func (s *OpenAIGatewayService) recheckSelectedOpenAIAccountFromDB(ctx context.Co
if err != nil || latest == nil {
return nil
}
if !isOpenAIAccountEligibleForRequest(latest, requestedModel, requireCompact) {
if !isOpenAIAccountEligibleForRequest(ctx, latest, requestedModel, requireCompact) {
return nil
}
if s.isOpenAIAccountRuntimeBlocked(latest) {
@@ -2067,8 +2067,12 @@ func (s *OpenAIGatewayService) shouldFailoverOpenAIUpstreamResponse(statusCode i
return isOpenAITransientProcessingError(statusCode, upstreamMsg, upstreamBody)
}
func (s *OpenAIGatewayService) handleFailoverSideEffects(ctx context.Context, resp *http.Response, account *Account) {
func (s *OpenAIGatewayService) handleFailoverSideEffects(ctx context.Context, resp *http.Response, account *Account, requestedModel ...string) {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
if len(requestedModel) > 0 {
s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, body, requestedModel[0])
return
}
s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, body)
}
@@ -2852,14 +2856,14 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco
Detail: upstreamDetail,
})
s.handleFailoverSideEffects(ctx, resp, account)
s.handleFailoverSideEffects(ctx, resp, account, upstreamModel)
return nil, &UpstreamFailoverError{
StatusCode: resp.StatusCode,
ResponseBody: respBody,
RetryableOnSameAccount: account.IsPoolMode() && (isPoolModeRetryableStatus(resp.StatusCode) || isOpenAITransientProcessingError(resp.StatusCode, upstreamMsg, respBody)),
}
}
return s.handleErrorResponse(ctx, resp, c, account, body)
return s.handleErrorResponse(ctx, resp, c, account, body, billingModel)
}
defer func() { _ = resp.Body.Close() }()
@@ -3344,7 +3348,8 @@ func (s *OpenAIGatewayService) handleFailoverErrorResponsePassthrough(
}
setOpsUpstreamError(c, resp.StatusCode, upstreamMsg, upstreamDetail)
logOpenAIInstructionsRequiredDebug(ctx, c, account, resp.StatusCode, upstreamMsg, requestBody, body)
_ = s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, body)
reqModel, _, _ := extractOpenAIRequestMetaFromBody(requestBody)
_ = s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, body, reqModel)
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
Platform: account.Platform,
AccountID: account.ID,
@@ -3387,7 +3392,8 @@ func (s *OpenAIGatewayService) handleErrorResponsePassthrough(
logOpenAIInstructionsRequiredDebug(ctx, c, account, resp.StatusCode, upstreamMsg, requestBody, body)
// 透传模式保留原始上游错误响应,但运行态账号状态仍需更新,
// 避免粘性路由继续复用刚被限流的账号。
_ = s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, body)
reqModel, _, _ := extractOpenAIRequestMetaFromBody(requestBody)
_ = s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, body, reqModel)
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
Platform: account.Platform,
AccountID: account.ID,
@@ -4061,6 +4067,7 @@ func (s *OpenAIGatewayService) handleErrorResponse(
c *gin.Context,
account *Account,
requestBody []byte,
requestedModel ...string,
) (*OpenAIForwardResult, error) {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
@@ -4137,7 +4144,14 @@ func (s *OpenAIGatewayService) handleErrorResponse(
}
// Handle upstream error (mark account status)
shouldDisable := s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, body)
var reqModel string
if len(requestedModel) > 0 {
reqModel = strings.TrimSpace(requestedModel[0])
}
if reqModel == "" {
reqModel, _, _ = extractOpenAIRequestMetaFromBody(requestBody)
}
shouldDisable := s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, body, reqModel)
kind := "http_error"
if shouldDisable {
kind = "failover"
@@ -4214,6 +4228,7 @@ func (s *OpenAIGatewayService) handleCompatErrorResponse(
c *gin.Context,
account *Account,
writeError compatErrorWriter,
requestedModel ...string,
) (*OpenAIForwardResult, error) {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
@@ -4269,8 +4284,12 @@ func (s *OpenAIGatewayService) handleCompatErrorResponse(
}
// Track rate limits and decide whether to trigger secondary failover.
var modelForCooldown string
if len(requestedModel) > 0 {
modelForCooldown = requestedModel[0]
}
shouldDisable := s.handleOpenAIAccountUpstreamError(
c.Request.Context(), account, resp.StatusCode, resp.Header, body,
c.Request.Context(), account, resp.StatusCode, resp.Header, body, modelForCooldown,
)
kind := "http_error"
if shouldDisable {