diff --git a/sdk/api/handlers/claude/code_handlers_cooldown_test.go b/sdk/api/handlers/claude/code_handlers_cooldown_test.go index 33c8e6e37f..c29353364b 100644 --- a/sdk/api/handlers/claude/code_handlers_cooldown_test.go +++ b/sdk/api/handlers/claude/code_handlers_cooldown_test.go @@ -224,3 +224,101 @@ func TestClaudeCooldownMessagesRoute(t *testing.T) { }) } } + +type cooldownClaudeTestExecutor struct { + preFTKeepAliveClaudeExecutor +} + +func (*cooldownClaudeTestExecutor) Identifier() string { return "claude" } + +func TestClaudeCooldownMessagesRoute_PassthroughUnifiedRateLimitHeaders(t *testing.T) { + gin.SetMode(gin.TestMode) + chunks := make(chan coreexecutor.StreamChunk) + close(chunks) + executor := &cooldownClaudeTestExecutor{preFTKeepAliveClaudeExecutor: preFTKeepAliveClaudeExecutor{chunks: chunks}} + manager := coreauth.NewManager(nil, coreauth.NewSessionAffinitySelector(&coreauth.RoundRobinSelector{}), nil) + manager.RegisterExecutor(executor) + const model = "issue-7417-cooldown-route-model" + const authID = "issue-7417-cooldown-route-auth" + next := time.Now().Add(7823 * time.Second) + auth := &coreauth.Auth{ + ID: authID, + Provider: executor.Identifier(), + Status: coreauth.StatusActive, + Unavailable: true, + NextRetryAfter: next, + Quota: coreauth.QuotaState{ + Exceeded: true, + Reason: "credential_quota", + NextRecoverAt: next, + Signals: map[string]string{ + "Anthropic-Ratelimit-Unified-Status": "rejected", + "Anthropic-Ratelimit-Unified-5h-Status": "rejected", + "Anthropic-Ratelimit-Unified-Reset": strconv.FormatInt(next.Add(-10*time.Second).Unix(), 10), + }, + }, + LastError: &coreauth.Error{Code: "rate_limit_error", Message: "upstream quota exceeded", HTTPStatus: http.StatusTooManyRequests}, + } + if _, errRegister := manager.Register(context.Background(), auth); errRegister != nil { + t.Fatalf("register auth: %v", errRegister) + } + registry.GetGlobalRegistry().RegisterClient(authID, executor.Identifier(), []*registry.ModelInfo{{ID: model}}) + t.Cleanup(func() { + registry.GetGlobalRegistry().UnregisterClient(authID) + }) + + for _, passthrough := range []bool{false, true} { + for _, streaming := range []bool{false, true} { + name := "passthrough_" + strconv.FormatBool(passthrough) + "_stream_" + strconv.FormatBool(streaming) + t.Run(name, func(t *testing.T) { + handler := NewClaudeCodeAPIHandler(handlers.NewBaseAPIHandlers(&sdkconfig.SDKConfig{PassthroughHeaders: passthrough}, manager)) + router := gin.New() + router.POST("/v1/messages", handler.ClaudeMessages) + + requestBody := `{"model":"` + model + `","messages":[{"role":"user","content":"hi"}],"stream":` + strconv.FormatBool(streaming) + `}` + request := httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(requestBody)) + request.Header.Set("Content-Type", "application/json") + recorder := httptest.NewRecorder() + + before := time.Now().Unix() + router.ServeHTTP(recorder, request) + after := time.Now().Unix() + + if recorder.Code != http.StatusTooManyRequests { + t.Fatalf("status = %d, want 429; body=%s", recorder.Code, recorder.Body.String()) + } + seconds, errSeconds := strconv.ParseInt(recorder.Header().Get("Retry-After"), 10, 64) + if errSeconds != nil || seconds <= 0 { + t.Fatalf("Retry-After = %q, want positive seconds", recorder.Header().Get("Retry-After")) + } + + if !passthrough { + if got := recorder.Header().Get("Anthropic-Ratelimit-Unified-Status"); got != "" { + t.Fatalf("passthrough=false leaked Anthropic-Ratelimit-Unified-Status = %q", got) + } + return + } + + if got := recorder.Header().Get("Anthropic-Ratelimit-Unified-Status"); got != "rejected" { + t.Fatalf("Anthropic-Ratelimit-Unified-Status = %q, want rejected", got) + } + if got := recorder.Header().Get("Anthropic-Ratelimit-Unified-Representative-Claim"); got != "five_hour" { + t.Fatalf("Anthropic-Ratelimit-Unified-Representative-Claim = %q, want five_hour", got) + } + if got := recorder.Header().Get("Anthropic-Ratelimit-Unified-Overage-Status"); got != "rejected" { + t.Fatalf("Anthropic-Ratelimit-Unified-Overage-Status = %q, want rejected", got) + } + if got := recorder.Header().Get("Anthropic-Ratelimit-Unified-5h-Status"); got != "rejected" { + t.Fatalf("Anthropic-Ratelimit-Unified-5h-Status = %q, want rejected", got) + } + resetAt, errReset := strconv.ParseInt(recorder.Header().Get("Anthropic-Ratelimit-Unified-Reset"), 10, 64) + if errReset != nil || resetAt < before+seconds || resetAt > after+seconds { + t.Fatalf("Anthropic-Ratelimit-Unified-Reset = %q, want unix epoch in [%d, %d]", recorder.Header().Get("Anthropic-Ratelimit-Unified-Reset"), before+seconds, after+seconds) + } + if got := recorder.Header().Get("Anthropic-Ratelimit-Unified-5h-Reset"); got != strconv.FormatInt(resetAt, 10) { + t.Fatalf("Anthropic-Ratelimit-Unified-5h-Reset = %q, want %d", got, resetAt) + } + }) + } + } +} diff --git a/sdk/cliproxy/auth/conductor_cooldown.go b/sdk/cliproxy/auth/conductor_cooldown.go index 8347f38ba2..7db44be3d5 100644 --- a/sdk/cliproxy/auth/conductor_cooldown.go +++ b/sdk/cliproxy/auth/conductor_cooldown.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "maps" "net" "net/http" "regexp" @@ -382,11 +383,18 @@ func (m *Manager) restoreCooldownRecordLocked(record CooldownStateRecord, now ti if quota.Exceeded && quota.NextRecoverAt.IsZero() { quota.NextRecoverAt = record.NextRetryAfter } + quota.cooldownEvaluated = quota.Exceeded + if executorKeyFromAuth(auth) == "claude" || strings.EqualFold(strings.TrimSpace(record.Provider), "claude") { + quota.cooldownSignals = sanitizePersistedClaudeCooldownSignals(record.CooldownSignals) + } if model == "" { auth.Unavailable = true auth.Status = StatusError auth.NextRetryAfter = record.NextRetryAfter + if len(quota.cooldownSignals) == 0 && auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(now) && !quota.NextRecoverAt.After(auth.Quota.NextRecoverAt) { + quota.cooldownSignals = auth.Quota.cooldownSignals + } applyCooldownFields(&auth.Quota, quota) auth.Quota = mergeQuotaObservation(auth.Quota, quota) auth.Generation++ @@ -453,10 +461,11 @@ func (m *Manager) restoreCooldownRecordLocked(record CooldownStateRecord, now ti } state.credentialQuotaPropagated = true applyCooldownFields(&state.Quota, QuotaState{ - Exceeded: true, - Reason: "credential_quota", - NextRecoverAt: auth.Quota.NextRecoverAt, - BackoffLevel: auth.Quota.BackoffLevel, + Exceeded: true, + Reason: "credential_quota", + NextRecoverAt: auth.Quota.NextRecoverAt, + BackoffLevel: auth.Quota.BackoffLevel, + cooldownSignals: auth.Quota.cooldownSignals, }) if state.NextRetryAfter.Before(auth.Quota.NextRecoverAt) { state.NextRetryAfter = auth.Quota.NextRecoverAt @@ -466,7 +475,12 @@ func (m *Manager) restoreCooldownRecordLocked(record CooldownStateRecord, now ti } else if isWrappedRecord { state.credentialQuotaPropagated = true if record.IndependentState != nil && (liveShadow == nil || !liveShadow.UpdatedAt.After(record.IndependentState.UpdatedAt)) { - state.credentialQuotaShadow = record.IndependentState.cloneShallow() + restoredShadow := record.IndependentState.cloneShallow() + restoredShadow.Quota.cooldownEvaluated = restoredShadow.Quota.Exceeded + if executorKeyFromAuth(auth) == "claude" || strings.EqualFold(strings.TrimSpace(record.Provider), "claude") { + restoredShadow.Quota.cooldownSignals = sanitizePersistedClaudeCooldownSignals(record.IndependentCooldownSignals) + } + state.credentialQuotaShadow = restoredShadow } else if liveShadow != nil { state.credentialQuotaShadow = liveShadow } @@ -756,6 +770,9 @@ func cooldownStateRecordEqual(a, b CooldownStateRecord) bool { if !cooldownErrorEqual(a.LastError, b.LastError) { return false } + if !maps.Equal(a.CooldownSignals, b.CooldownSignals) || !maps.Equal(a.IndependentCooldownSignals, b.IndependentCooldownSignals) { + return false + } return cooldownIndependentStateEqual(a.IndependentState, b.IndependentState) } @@ -793,16 +810,21 @@ func authCooldownStateRecord(auth *Auth, now time.Time) (CooldownStateRecord, bo if auth == nil || !auth.Unavailable || auth.NextRetryAfter.IsZero() || !auth.NextRetryAfter.After(now) { return CooldownStateRecord{}, false } + var cooldownSignals map[string]string + if auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(now) && !auth.NextRetryAfter.After(auth.Quota.NextRecoverAt) { + cooldownSignals = sanitizePersistedClaudeCooldownSignals(auth.Quota.cooldownSignals) + } return CooldownStateRecord{ - Provider: strings.TrimSpace(auth.Provider), - AuthID: auth.ID, - AuthFile: cooldownAuthFile(auth), - Status: "cooling", - NextRetryAfter: auth.NextRetryAfter, - Reason: cooldownReason(auth.StatusMessage, auth.Quota, auth.LastError), - Quota: cooldownFieldsOf(auth.Quota), - LastError: cloneError(auth.LastError), - UpdatedAt: auth.UpdatedAt, + Provider: strings.TrimSpace(auth.Provider), + AuthID: auth.ID, + AuthFile: cooldownAuthFile(auth), + Status: "cooling", + NextRetryAfter: auth.NextRetryAfter, + Reason: cooldownReason(auth.StatusMessage, auth.Quota, auth.LastError), + Quota: cooldownFieldsOf(auth.Quota), + LastError: cloneError(auth.LastError), + UpdatedAt: auth.UpdatedAt, + CooldownSignals: cooldownSignals, }, true } @@ -811,20 +833,32 @@ func modelCooldownStateRecord(auth *Auth, model string, state *ModelState, now t if auth == nil || state == nil || model == "" || !state.Unavailable || state.NextRetryAfter.IsZero() || !state.NextRetryAfter.After(now) { return CooldownStateRecord{}, false } + var cooldownSignals map[string]string + liveCredentialWrap := auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now) + expiredCredentialWrap := (state.credentialQuotaPropagated || state.Quota.Reason == "credential_quota") && !liveCredentialWrap + if !expiredCredentialWrap && state.Quota.Exceeded && state.Quota.NextRecoverAt.After(now) && + (state.Quota.Reason == "credential_quota" || !state.NextRetryAfter.After(state.Quota.NextRecoverAt)) { + cooldownSignals = sanitizePersistedClaudeCooldownSignals(state.Quota.cooldownSignals) + } record := CooldownStateRecord{ - Provider: strings.TrimSpace(auth.Provider), - AuthID: auth.ID, - AuthFile: cooldownAuthFile(auth), - Model: model, - Status: "cooling", - NextRetryAfter: state.NextRetryAfter, - Reason: cooldownReason(state.StatusMessage, state.Quota, state.LastError), - Quota: cooldownFieldsOf(state.Quota), - LastError: cloneError(state.LastError), - UpdatedAt: state.UpdatedAt, + Provider: strings.TrimSpace(auth.Provider), + AuthID: auth.ID, + AuthFile: cooldownAuthFile(auth), + Model: model, + Status: "cooling", + NextRetryAfter: state.NextRetryAfter, + Reason: cooldownReason(state.StatusMessage, state.Quota, state.LastError), + Quota: cooldownFieldsOf(state.Quota), + LastError: cloneError(state.LastError), + UpdatedAt: state.UpdatedAt, + CooldownSignals: cooldownSignals, } if state.credentialQuotaShadow != nil { record.IndependentState = state.credentialQuotaShadow.cloneShallow() + if state.credentialQuotaShadow.Quota.Exceeded && state.credentialQuotaShadow.Quota.NextRecoverAt.After(now) && + !state.credentialQuotaShadow.NextRetryAfter.After(state.credentialQuotaShadow.Quota.NextRecoverAt) { + record.IndependentCooldownSignals = sanitizePersistedClaudeCooldownSignals(state.credentialQuotaShadow.Quota.cooldownSignals) + } } record.CredentialQuotaPropagated = state.credentialQuotaPropagated return record, true @@ -1123,8 +1157,12 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) { state.NextRetryAfter = next } case 429: + prevStateQuotaActive := state.Quota.Exceeded && state.Quota.NextRecoverAt.After(now) + prevStateRecoverAt := state.Quota.NextRecoverAt + prevStateCooldownSignals := maps.Clone(state.Quota.cooldownSignals) var next time.Time var credentialNext time.Time + keptPriorStateQuota := false backoffLevel := state.Quota.BackoffLevel if result.CredentialScope { backoffLevel = 0 @@ -1152,16 +1190,32 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) { next, backoffLevel = quotaCooldownAfterFailure(quotaForFailure, now) } credentialNext = next - if state.Quota.Exceeded && state.Quota.NextRecoverAt.After(next) { - next = state.Quota.NextRecoverAt + if prevStateQuotaActive && (prevStateRecoverAt.After(next) || (!result.CredentialScope && result.RetryAfter == nil)) { + next = prevStateRecoverAt + keptPriorStateQuota = true + } else if result.CredentialScope && result.RetryAfter == nil && auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now) { + // quotaCooldownAfterFailure reused the open credential_quota window. + keptPriorStateQuota = prevStateQuotaActive && prevStateRecoverAt.Equal(next) + } + } + var cooldownSignals map[string]string + if !disableCooling { + if keptPriorStateQuota { + cooldownSignals = prevStateCooldownSignals + } else { + cooldownSignals = rejectedClaudeSignalsForCooldown(result.Provider, responseHeaders) + if len(cooldownSignals) == 0 && prevStateQuotaActive && prevStateRecoverAt.Equal(next) { + cooldownSignals = prevStateCooldownSignals + } } } state.NextRetryAfter = next applyCooldownFields(&state.Quota, QuotaState{ - Exceeded: true, - Reason: "quota", - NextRecoverAt: next, - BackoffLevel: backoffLevel, + Exceeded: true, + Reason: "quota", + NextRecoverAt: next, + BackoffLevel: backoffLevel, + cooldownSignals: cooldownSignals, }) // Non-credential own failures re-derive their independent layer // below, so a stale shadow is dropped. A credential-scoped @@ -1187,12 +1241,24 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) { } } if result.CredentialScope && !disableCooling { + prevAuthCredActive := auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now) + prevAuthCredRecoverAt := auth.Quota.NextRecoverAt + prevAuthCredSignals := maps.Clone(auth.Quota.cooldownSignals) + var credCooldownSignals map[string]string + if prevAuthCredActive && (prevAuthCredRecoverAt.After(credentialNext) || result.RetryAfter == nil) { + credCooldownSignals = prevAuthCredSignals + } else { + credCooldownSignals = rejectedClaudeSignalsForCooldown(result.Provider, responseHeaders) + if len(credCooldownSignals) == 0 && prevAuthCredActive && prevAuthCredRecoverAt.Equal(credentialNext) { + credCooldownSignals = prevAuthCredSignals + } + } for _, otherState := range auth.ModelStates { if otherState != nil && otherState != state { // Normalize a naturally expired wrap first (issue #6690) so a // stale marker cannot suppress snapshotting a fresh block. if (otherState.credentialQuotaPropagated || otherState.Quota.Reason == "credential_quota") && - !(auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now)) { + !prevAuthCredActive { if shadow := otherState.credentialQuotaShadow; shadow != nil && isModelStateActiveCooldown(shadow, now) { preservedSeq := shadow.cooldownRecordedSeq *otherState = *shadow.cloneShallow() @@ -1219,8 +1285,13 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) { otherState.Unavailable = true otherState.Status = StatusError otherQuotaNext := credentialNext + otherSignals := credCooldownSignals + if prevAuthCredActive && prevAuthCredRecoverAt.After(otherQuotaNext) { + otherQuotaNext = prevAuthCredRecoverAt + } if otherState.Quota.Exceeded && otherState.Quota.NextRecoverAt.After(otherQuotaNext) { otherQuotaNext = otherState.Quota.NextRecoverAt + otherSignals = maps.Clone(otherState.Quota.cooldownSignals) } otherRetryAfter := otherQuotaNext // Propagation only extends a sibling's still-live @@ -1230,25 +1301,34 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) { } otherState.NextRetryAfter = otherRetryAfter applyCooldownFields(&otherState.Quota, QuotaState{ - Exceeded: true, - Reason: "credential_quota", - NextRecoverAt: otherQuotaNext, - BackoffLevel: backoffLevel, + Exceeded: true, + Reason: "credential_quota", + NextRecoverAt: otherQuotaNext, + BackoffLevel: backoffLevel, + cooldownSignals: otherSignals, }) } } if !wasTerminalUnauthorized { auth.Unavailable = true authNext := credentialNext - if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && - auth.Quota.NextRecoverAt.After(authNext) { - authNext = auth.Quota.NextRecoverAt + if prevAuthCredActive && prevAuthCredRecoverAt.After(authNext) { + authNext = prevAuthCredRecoverAt } auth.Quota.Exceeded = true auth.Quota.Reason = "credential_quota" auth.Quota.NextRecoverAt = authNext auth.Quota.BackoffLevel = backoffLevel + auth.Quota.cooldownSignals = maps.Clone(credCooldownSignals) + auth.Quota.cooldownEvaluated = true auth.NextRetryAfter = authNext + if authNext.After(state.Quota.NextRecoverAt) || (prevAuthCredActive && result.RetryAfter == nil) { + state.Quota.NextRecoverAt = authNext + state.Quota.cooldownSignals = maps.Clone(credCooldownSignals) + if state.NextRetryAfter.Before(authNext) { + state.NextRetryAfter = authNext + } + } // The reporting model is now part of the wrap too: mark it // so a later credential report from another model cannot // snapshot this credential-derived state as independent. @@ -1310,10 +1390,11 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) { } if !(outer.Quota.Exceeded && outer.Quota.Reason == "credential_quota") { applyCooldownFields(&outer.Quota, QuotaState{ - Exceeded: true, - Reason: "credential_quota", - NextRecoverAt: auth.Quota.NextRecoverAt, - BackoffLevel: auth.Quota.BackoffLevel, + Exceeded: true, + Reason: "credential_quota", + NextRecoverAt: auth.Quota.NextRecoverAt, + BackoffLevel: auth.Quota.BackoffLevel, + cooldownSignals: auth.Quota.cooldownSignals, }) } *outerState = *outer @@ -1337,7 +1418,36 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) { disableCooling = false } if !wasTerminalUnauthorized { + prevAuthQuotaActive := auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(now) + prevAuthRecoverAt := auth.Quota.NextRecoverAt + prevAuthCooldownSignals := maps.Clone(auth.Quota.cooldownSignals) + keptPriorAuthQuota := false + if !disableCooling && statusCodeFromResult(result.Error) == 429 && prevAuthQuotaActive { + if result.RetryAfter == nil { + keptPriorAuthQuota = true + } else { + cooldown := *result.RetryAfter + if cooldown < minQuotaCooldownFloor { + cooldown = minQuotaCooldownFloor + } + if prevAuthRecoverAt.After(now.Add(cooldown).Round(0)) { + keptPriorAuthQuota = true + } + } + } applyAuthFailureState(auth, result.Error, result.RetryAfter, now, disableCooling) + if !disableCooling && statusCodeFromResult(result.Error) == 429 && auth.Quota.Exceeded { + auth.Quota.cooldownEvaluated = true + if keptPriorAuthQuota { + auth.Quota.cooldownSignals = prevAuthCooldownSignals + } else if rejectedSignals := rejectedClaudeSignalsForCooldown(result.Provider, responseHeaders); len(rejectedSignals) > 0 { + auth.Quota.cooldownSignals = rejectedSignals + } else if prevAuthQuotaActive && auth.Quota.NextRecoverAt.Equal(prevAuthRecoverAt) { + auth.Quota.cooldownSignals = prevAuthCooldownSignals + } else { + auth.Quota.cooldownSignals = nil + } + } auth.cooldownRecordedSeq = m.currentExecutionSeq() } } @@ -1630,15 +1740,25 @@ func mergeModelState(target, source *ModelState) *ModelState { NextRetryAfter: target.NextRetryAfter, LastError: cloneError(preferred.LastError), Quota: QuotaState{ - Exceeded: target.Quota.Exceeded || source.Quota.Exceeded, - Reason: preferred.Quota.Reason, - NextRecoverAt: target.Quota.NextRecoverAt, - BackoffLevel: target.Quota.BackoffLevel, + Exceeded: target.Quota.Exceeded || source.Quota.Exceeded, + Reason: preferred.Quota.Reason, + NextRecoverAt: target.Quota.NextRecoverAt, + BackoffLevel: target.Quota.BackoffLevel, + cooldownEvaluated: target.Quota.cooldownEvaluated || source.Quota.cooldownEvaluated, }, UpdatedAt: target.UpdatedAt, } merged.Quota = mergeQuotaObservation(merged.Quota, fallback.Quota) merged.Quota = mergeQuotaObservation(merged.Quota, preferred.Quota) + if source.Quota.NextRecoverAt.After(target.Quota.NextRecoverAt) { + merged.Quota.cooldownSignals = maps.Clone(source.Quota.cooldownSignals) + } else if target.Quota.NextRecoverAt.After(source.Quota.NextRecoverAt) { + merged.Quota.cooldownSignals = maps.Clone(target.Quota.cooldownSignals) + } else if len(preferred.Quota.cooldownSignals) > 0 { + merged.Quota.cooldownSignals = maps.Clone(preferred.Quota.cooldownSignals) + } else if len(fallback.Quota.cooldownSignals) > 0 { + merged.Quota.cooldownSignals = maps.Clone(fallback.Quota.cooldownSignals) + } if source.NextRetryAfter.After(merged.NextRetryAfter) { merged.NextRetryAfter = source.NextRetryAfter } @@ -1873,10 +1993,7 @@ func updateAggregatedAvailability(auth *Auth, now time.Time) { } else if auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(now) { // Retain active auth-level quota cooldown } else { - auth.Quota.Exceeded = false - auth.Quota.Reason = "" - auth.Quota.NextRecoverAt = time.Time{} - auth.Quota.BackoffLevel = 0 + applyCooldownFields(&auth.Quota, QuotaState{}) } } @@ -1920,10 +2037,7 @@ func clearAuthStateOnSuccess(auth *Auth, now time.Time) { auth.Unavailable = false auth.Status = StatusActive auth.StatusMessage = "" - auth.Quota.Exceeded = false - auth.Quota.Reason = "" - auth.Quota.NextRecoverAt = time.Time{} - auth.Quota.BackoffLevel = 0 + applyCooldownFields(&auth.Quota, QuotaState{}) auth.LastError = nil auth.NextRetryAfter = time.Time{} auth.cooldownRecordedSeq = 0 diff --git a/sdk/cliproxy/auth/conductor_selection.go b/sdk/cliproxy/auth/conductor_selection.go index cd37a04a80..a3096ed276 100644 --- a/sdk/cliproxy/auth/conductor_selection.go +++ b/sdk/cliproxy/auth/conductor_selection.go @@ -909,7 +909,10 @@ func (m *Manager) availableAuthsForRouteModelForRequest(ctx context.Context, aut if resetIn < 0 { resetIn = 0 } - return nil, newModelCooldownErrorWithCause(routeModel, providerForError, resetIn, lastCandidateErr) + claudeCooldown, claudeSignals := claudeCooldownMetadataForAuths(ctx, auths, func(candidate *Auth) string { + return m.selectionModelForAuth(candidate, routeModel) + }, now, clientHeaders...) + return nil, newModelCooldownErrorForCandidates(routeModel, providerForError, resetIn, lastCandidateErr, claudeCooldown, claudeSignals) } if unauthorizedCount == len(auths) && len(auths) > 0 { terminalCause := latestUnauthorizedCandidateError(auths) @@ -1024,7 +1027,14 @@ func restoreModelCooldownErrorModel(err error, requestedModel string) error { if !errors.As(err, &cooldownErr) || cooldownErr == nil || cooldownErr.model != "" { return err } - return newModelCooldownErrorWithCause(requestedModel, cooldownErr.provider, cooldownErr.resetIn, cooldownErr.cause) + return newModelCooldownErrorForCandidates( + requestedModel, + cooldownErr.provider, + cooldownErr.resetIn, + cooldownErr.cause, + cooldownErr.claudeCooldown, + cooldownErr.claudeSignals, + ) } func latestUnauthorizedCandidateError(auths []*Auth) error { diff --git a/sdk/cliproxy/auth/conductor_selection_cooldown_test.go b/sdk/cliproxy/auth/conductor_selection_cooldown_test.go index 95217a9870..a63dcda453 100644 --- a/sdk/cliproxy/auth/conductor_selection_cooldown_test.go +++ b/sdk/cliproxy/auth/conductor_selection_cooldown_test.go @@ -3,10 +3,13 @@ package auth import ( "context" "errors" + "net/http" + "strconv" "strings" "testing" "time" + internallogging "github.com/router-for-me/CLIProxyAPI/v8/internal/logging" "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" ) @@ -813,3 +816,1197 @@ func TestErrorWithCause_ErrorDirectFormat(t *testing.T) { t.Fatalf("Error() = %q, want containing upstream error summary", got) } } + +func TestModelCooldownError_ClaudeUnifiedRateLimitHeaders(t *testing.T) { + t.Parallel() + + t.Run("preserves observed 7d claim under 5h remaining across affinity and scheduler paths", func(t *testing.T) { + for _, useAffinity := range []bool{true, false} { + name := "scheduler" + if useAffinity { + name = "session_affinity" + } + t.Run(name, func(t *testing.T) { + var selector Selector + if useAffinity { + selector = NewSessionAffinitySelector(&RoundRobinSelector{}) + } + manager := NewManager(nil, selector, nil) + manager.RegisterExecutor(&mockCustomErrorExecutor{identifier: "claude"}) + + const model = "issue-7417-claude-7d-short-" + modelID := model + name + authID := "issue-7417-auth-" + name + registry.GetGlobalRegistry().RegisterClient(authID, "claude", []*registry.ModelInfo{{ID: modelID}}) + t.Cleanup(func() { + registry.GetGlobalRegistry().UnregisterClient(authID) + }) + + if _, errRegister := manager.Register(context.Background(), &Auth{ + ID: authID, + Provider: "claude", + Status: StatusActive, + }); errRegister != nil { + t.Fatalf("register auth: %v", errRegister) + } + + retryDelay := 2 * time.Hour + upstreamReset := time.Now().Add(retryDelay - 15*time.Second).Unix() + markCtx := internallogging.WithResponseHeadersHolder(context.Background()) + internallogging.SetResponseHeaders(markCtx, http.Header{ + "Anthropic-Ratelimit-Unified-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-7d-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-7d-Reset": {strconv.FormatInt(upstreamReset, 10)}, + "Anthropic-Ratelimit-Unified-5h-Status": {"allowed"}, + "Anthropic-Ratelimit-Unified-Representative-Claim": {"seven_day"}, + "Anthropic-Ratelimit-Unified-Overage-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-Reset": {strconv.FormatInt(upstreamReset, 10)}, + }) + manager.MarkResult(markCtx, Result{ + AuthID: authID, + Provider: "claude", + Model: modelID, + Success: false, + CredentialScope: true, + RetryAfter: &retryDelay, + Error: &Error{Code: ErrorCodeCredentialQuota, Message: "rate_limit_error", HTTPStatus: http.StatusTooManyRequests}, + }) + + _, errExecute := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: modelID}, cliproxyexecutor.Options{}) + if errExecute == nil { + t.Fatal("expected cooldown error, got nil") + } + + var cooldownErr *modelCooldownError + if !errors.As(errExecute, &cooldownErr) { + t.Fatalf("Execute() error = %T (%v), want *modelCooldownError", errExecute, errExecute) + } + + before := time.Now().Unix() + headers := cooldownErr.Headers() + after := time.Now().Unix() + retrySeconds, errSeconds := strconv.ParseInt(headers.Get("Retry-After"), 10, 64) + if errSeconds != nil || retrySeconds <= 0 { + t.Fatalf("Retry-After = %q, want positive seconds", headers.Get("Retry-After")) + } + if got := headers.Get("Anthropic-Ratelimit-Unified-Status"); got != "rejected" { + t.Fatalf("Anthropic-Ratelimit-Unified-Status = %q, want rejected", got) + } + if got := headers.Get("Anthropic-Ratelimit-Unified-Representative-Claim"); got != "seven_day" { + t.Fatalf("Anthropic-Ratelimit-Unified-Representative-Claim = %q, want seven_day", got) + } + if got := headers.Get("Anthropic-Ratelimit-Unified-7d-Status"); got != "rejected" { + t.Fatalf("Anthropic-Ratelimit-Unified-7d-Status = %q, want rejected", got) + } + if got := headers.Get("Anthropic-Ratelimit-Unified-5h-Status"); got != "allowed" { + t.Fatalf("Anthropic-Ratelimit-Unified-5h-Status = %q, want allowed", got) + } + if got := headers.Get("Anthropic-Ratelimit-Unified-Overage-Status"); got != "rejected" { + t.Fatalf("Anthropic-Ratelimit-Unified-Overage-Status = %q, want rejected", got) + } + + unifiedReset, errReset := strconv.ParseInt(headers.Get("Anthropic-Ratelimit-Unified-Reset"), 10, 64) + if errReset != nil { + t.Fatalf("Anthropic-Ratelimit-Unified-Reset = %q, want unix epoch", headers.Get("Anthropic-Ratelimit-Unified-Reset")) + } + if unifiedReset < before+retrySeconds || unifiedReset > after+retrySeconds { + t.Fatalf("Anthropic-Ratelimit-Unified-Reset = %d, want in [%d, %d] aligned with Retry-After %d", unifiedReset, before+retrySeconds, after+retrySeconds, retrySeconds) + } + if got := headers.Get("Anthropic-Ratelimit-Unified-7d-Reset"); got != strconv.FormatInt(unifiedReset, 10) { + t.Fatalf("Anthropic-Ratelimit-Unified-7d-Reset = %q, want %d", got, unifiedReset) + } + + safeHeaders := SafeResponseHeaders(errExecute) + if got := safeHeaders.Get("Anthropic-Ratelimit-Unified-Status"); got != "" { + t.Fatalf("SafeResponseHeaders leaked unified header %q", got) + } + }) + } + }) + + t.Run("infers claim from duration when signals are absent", func(t *testing.T) { + errFiveHour := newModelCooldownError("claude-haiku-4-5-20251001", "claude", 2*time.Hour) + h5 := errFiveHour.Headers() + if got := h5.Get("Anthropic-Ratelimit-Unified-Representative-Claim"); got != "five_hour" { + t.Fatalf("5h fallback claim = %q, want five_hour", got) + } + if got := h5.Get("Anthropic-Ratelimit-Unified-5h-Status"); got != "rejected" { + t.Fatalf("5h fallback status = %q, want rejected", got) + } + if got := h5.Get("Anthropic-Ratelimit-Unified-Overage-Status"); got != "rejected" { + t.Fatalf("5h fallback overage status = %q, want rejected", got) + } + + errSevenDay := newModelCooldownError("claude-haiku-4-5-20251001", "claude", 6*time.Hour) + h7 := errSevenDay.Headers() + if got := h7.Get("Anthropic-Ratelimit-Unified-Representative-Claim"); got != "seven_day" { + t.Fatalf("7d fallback claim = %q, want seven_day", got) + } + if got := h7.Get("Anthropic-Ratelimit-Unified-7d-Status"); got != "rejected" { + t.Fatalf("7d fallback status = %q, want rejected", got) + } + }) + + t.Run("preserves allowed shared windows on overage claim and aligns model-family window resets", func(t *testing.T) { + overageErr := newModelCooldownErrorForCandidates("claude-sonnet-4-5-20250929", "claude", 30*time.Second, nil, true, map[string]string{ + "Anthropic-Ratelimit-Unified-Status": "rejected", + "Anthropic-Ratelimit-Unified-Representative-Claim": "overage", + "Anthropic-Ratelimit-Unified-5h-Status": "allowed", + "Anthropic-Ratelimit-Unified-7d-Status": "allowed", + "Anthropic-Ratelimit-Unified-Overage-Status": "rejected", + }) + overageHeaders := overageErr.Headers() + if got := overageHeaders.Get("Anthropic-Ratelimit-Unified-5h-Status"); got != "allowed" { + t.Fatalf("overage cooldown overwrote 5h status = %q, want allowed", got) + } + if got := overageHeaders.Get("Anthropic-Ratelimit-Unified-7d-Status"); got != "allowed" { + t.Fatalf("overage cooldown overwrote 7d status = %q, want allowed", got) + } + if got := overageHeaders.Get("Anthropic-Ratelimit-Unified-Representative-Claim"); got != "overage" { + t.Fatalf("overage cooldown claim = %q, want overage", got) + } + + sonnetErr := newModelCooldownErrorForCandidates("claude-sonnet-4-5-20250929", "claude", 2*time.Hour, nil, true, map[string]string{ + "Anthropic-Ratelimit-Unified-Status": "rejected", + "Anthropic-Ratelimit-Unified-Representative-Claim": "seven_day_sonnet", + "Anthropic-Ratelimit-Unified-5h-Status": "allowed", + "Anthropic-Ratelimit-Unified-7d-Status": "allowed", + "Anthropic-Ratelimit-Unified-7d_sonnet-Status": "rejected", + "Anthropic-Ratelimit-Unified-7d_sonnet-Reset": "1700000000", + "Anthropic-Ratelimit-Unified-Overage-Status": "rejected", + }) + sonnetHeaders := sonnetErr.Headers() + if got := sonnetHeaders.Get("Anthropic-Ratelimit-Unified-5h-Status"); got != "allowed" { + t.Fatalf("7d_sonnet cooldown overwrote 5h status = %q, want allowed", got) + } + if got := sonnetHeaders.Get("Anthropic-Ratelimit-Unified-7d_sonnet-Reset"); got != sonnetHeaders.Get("Anthropic-Ratelimit-Unified-Reset") { + t.Fatalf("7d_sonnet reset = %q, want %q", got, sonnetHeaders.Get("Anthropic-Ratelimit-Unified-Reset")) + } + + overageNoClaimErr := newModelCooldownErrorForCandidates("claude-sonnet-4-5-20250929", "claude", 30*time.Second, nil, true, map[string]string{ + "Anthropic-Ratelimit-Unified-Status": "rejected", + "Anthropic-Ratelimit-Unified-5h-Status": "allowed", + "Anthropic-Ratelimit-Unified-7d-Status": "allowed", + "Anthropic-Ratelimit-Unified-Overage-Status": "rejected", + }) + overageNoClaimHeaders := overageNoClaimErr.Headers() + if got := overageNoClaimHeaders.Get("Anthropic-Ratelimit-Unified-Representative-Claim"); got != "overage" { + t.Fatalf("overage without claim inferred %q, want overage", got) + } + if got := overageNoClaimHeaders.Get("Anthropic-Ratelimit-Unified-5h-Status"); got != "allowed" { + t.Fatalf("overage without claim 5h status = %q, want allowed", got) + } + + sonnetNoClaimErr := newModelCooldownErrorForCandidates("claude-sonnet-4-5-20250929", "claude", 2*time.Hour, nil, true, map[string]string{ + "Anthropic-Ratelimit-Unified-Status": "rejected", + "Anthropic-Ratelimit-Unified-5h-Status": "allowed", + "Anthropic-Ratelimit-Unified-7d-Status": "allowed", + "Anthropic-Ratelimit-Unified-7d_sonnet-Status": "rejected", + "Anthropic-Ratelimit-Unified-Overage-Status": "rejected", + }) + sonnetNoClaimHeaders := sonnetNoClaimErr.Headers() + if got := sonnetNoClaimHeaders.Get("Anthropic-Ratelimit-Unified-Representative-Claim"); got != "seven_day_sonnet" { + t.Fatalf("7d_sonnet without claim inferred %q, want seven_day_sonnet", got) + } + + fiveAllowedOnlyErr := newModelCooldownErrorForCandidates("claude-sonnet-4-5-20250929", "claude", 2*time.Hour, nil, true, map[string]string{ + "Anthropic-Ratelimit-Unified-Status": "rejected", + "Anthropic-Ratelimit-Unified-5h-Status": "allowed", + }) + fiveAllowedOnlyHeaders := fiveAllowedOnlyErr.Headers() + if got := fiveAllowedOnlyHeaders.Get("Anthropic-Ratelimit-Unified-Representative-Claim"); got != "seven_day" { + t.Fatalf("5h=allowed with 2h reset inferred %q, want seven_day", got) + } + if got := fiveAllowedOnlyHeaders.Get("Anthropic-Ratelimit-Unified-7d-Status"); got != "rejected" { + t.Fatalf("5h=allowed with 2h reset 7d status = %q, want rejected", got) + } + }) + + t.Run("does not emit unified headers for non-Claude or cloudflare cooldowns", func(t *testing.T) { + manager := NewManager(nil, NewSessionAffinitySelector(&RoundRobinSelector{}), nil) + manager.RegisterExecutor(&mockCustomErrorExecutor{identifier: "codex"}) + manager.RegisterExecutor(&mockCustomErrorExecutor{identifier: "claude"}) + + const codexModel = "issue-7417-codex-model" + const codexAuthID = "issue-7417-codex-auth" + const cfModel = "issue-7417-cf-model" + const cfAuthID = "issue-7417-cf-auth" + next := time.Now().Add(30 * time.Minute) + + registry.GetGlobalRegistry().RegisterClient(codexAuthID, "codex", []*registry.ModelInfo{{ID: codexModel}}) + registry.GetGlobalRegistry().RegisterClient(cfAuthID, "claude", []*registry.ModelInfo{{ID: cfModel}}) + t.Cleanup(func() { + registry.GetGlobalRegistry().UnregisterClient(codexAuthID) + registry.GetGlobalRegistry().UnregisterClient(cfAuthID) + }) + + if _, errRegister := manager.Register(context.Background(), &Auth{ + ID: codexAuthID, + Provider: "codex", + Status: StatusActive, + Unavailable: true, + NextRetryAfter: next, + Quota: QuotaState{Exceeded: true, Reason: "quota", NextRecoverAt: next}, + }); errRegister != nil { + t.Fatalf("register codex auth: %v", errRegister) + } + if _, errRegister := manager.Register(context.Background(), &Auth{ + ID: cfAuthID, + Provider: "claude", + Status: StatusActive, + Unavailable: true, + NextRetryAfter: next, + ModelStates: map[string]*ModelState{ + cfModel: { + Status: StatusError, + Unavailable: true, + NextRetryAfter: next, + Quota: QuotaState{Exceeded: true, Reason: "cloudflare challenge", NextRecoverAt: next}, + }, + }, + }); errRegister != nil { + t.Fatalf("register cloudflare auth: %v", errRegister) + } + + _, errCodex := manager.Execute(context.Background(), []string{"codex"}, cliproxyexecutor.Request{Model: codexModel}, cliproxyexecutor.Options{}) + var codexCooldown *modelCooldownError + if !errors.As(errCodex, &codexCooldown) { + t.Fatalf("codex Execute() error = %T (%v), want *modelCooldownError", errCodex, errCodex) + } + if codexCooldown.provider != "" { + t.Fatalf("codex cooldown provider = %q, want empty mixed provider", codexCooldown.provider) + } + if got := codexCooldown.Headers().Get("Anthropic-Ratelimit-Unified-Status"); got != "" { + t.Fatalf("codex cooldown leaked Anthropic-Ratelimit-Unified-Status = %q", got) + } + + _, errCF := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: cfModel}, cliproxyexecutor.Options{}) + var cfCooldown *modelCooldownError + if !errors.As(errCF, &cfCooldown) { + t.Fatalf("cloudflare Execute() error = %T (%v), want *modelCooldownError", errCF, errCF) + } + if got := cfCooldown.Headers().Get("Anthropic-Ratelimit-Unified-Status"); got != "" { + t.Fatalf("cloudflare challenge cooldown leaked Anthropic-Ratelimit-Unified-Status = %q", got) + } + }) + + t.Run("does not synthesize unified headers for ordinary per-minute rate limits across affinity and scheduler paths", func(t *testing.T) { + for _, useAffinity := range []bool{true, false} { + pathName := "scheduler" + if useAffinity { + pathName = "session_affinity" + } + t.Run(pathName, func(t *testing.T) { + var selector Selector + if useAffinity { + selector = NewSessionAffinitySelector(&RoundRobinSelector{}) + } + manager := NewManager(nil, selector, nil) + manager.RegisterExecutor(&mockCustomErrorExecutor{identifier: "claude"}) + + allowedWindowModel := "issue-7417-rpm-allowed-" + pathName + allowedWindowAuth := "issue-7417-rpm-allowed-auth-" + pathName + plainRPMModel := "issue-7417-rpm-plain-" + pathName + plainRPMAuth := "issue-7417-rpm-plain-auth-" + pathName + modelQuotaModel := "issue-7417-model-unified-" + pathName + modelQuotaAuth := "issue-7417-model-unified-auth-" + pathName + + registry.GetGlobalRegistry().RegisterClient(allowedWindowAuth, "claude", []*registry.ModelInfo{{ID: allowedWindowModel}}) + registry.GetGlobalRegistry().RegisterClient(plainRPMAuth, "claude", []*registry.ModelInfo{{ID: plainRPMModel}}) + registry.GetGlobalRegistry().RegisterClient(modelQuotaAuth, "claude", []*registry.ModelInfo{{ID: modelQuotaModel}}) + t.Cleanup(func() { + registry.GetGlobalRegistry().UnregisterClient(allowedWindowAuth) + registry.GetGlobalRegistry().UnregisterClient(plainRPMAuth) + registry.GetGlobalRegistry().UnregisterClient(modelQuotaAuth) + }) + + for _, id := range []string{allowedWindowAuth, plainRPMAuth, modelQuotaAuth} { + if _, errRegister := manager.Register(context.Background(), &Auth{ + ID: id, + Provider: "claude", + Status: StatusActive, + }); errRegister != nil { + t.Fatalf("register %s: %v", id, errRegister) + } + } + + shortRetry := 30 * time.Second + rpmBody := `{"type":"error","error":{"type":"rate_limit_error","message":"This request would exceed the requests per minute rate limit. Please try again later."}}` + + // 1. Per-minute 429 with allowed 5h window and future unified reset. + allowedCtx := internallogging.WithResponseHeadersHolder(context.Background()) + internallogging.SetResponseHeaders(allowedCtx, http.Header{ + "Retry-After": {"30"}, + "Anthropic-Ratelimit-Unified-5h-Status": {"allowed"}, + "Anthropic-Ratelimit-Unified-Reset": {strconv.FormatInt(time.Now().Add(4*time.Hour).Unix(), 10)}, + }) + manager.MarkResult(allowedCtx, Result{ + AuthID: allowedWindowAuth, + Provider: "claude", + Model: allowedWindowModel, + Success: false, + CredentialScope: false, + RetryAfter: &shortRetry, + Error: &Error{Code: "rate_limit_error", Message: rpmBody, HTTPStatus: http.StatusTooManyRequests}, + }) + + // 2. Per-minute 429 with only Retry-After and no unified rejection headers. + plainCtx := internallogging.WithResponseHeadersHolder(context.Background()) + internallogging.SetResponseHeaders(plainCtx, http.Header{ + "Retry-After": {"30"}, + }) + manager.MarkResult(plainCtx, Result{ + AuthID: plainRPMAuth, + Provider: "claude", + Model: plainRPMModel, + Success: false, + CredentialScope: false, + RetryAfter: &shortRetry, + Error: &Error{Code: "rate_limit_error", Message: rpmBody, HTTPStatus: http.StatusTooManyRequests}, + }) + + // 3. Model-level cooling (e.g. API-key upstream) that explicitly reports unified 5h rejection. + fiveHourDelay := 2 * time.Hour + rejectedCtx := internallogging.WithResponseHeadersHolder(context.Background()) + internallogging.SetResponseHeaders(rejectedCtx, http.Header{ + "Retry-After": {"7200"}, + "Anthropic-Ratelimit-Unified-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-5h-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-Reset": {strconv.FormatInt(time.Now().Add(fiveHourDelay).Unix(), 10)}, + }) + manager.MarkResult(rejectedCtx, Result{ + AuthID: modelQuotaAuth, + Provider: "claude", + Model: modelQuotaModel, + Success: false, + CredentialScope: false, + RetryAfter: &fiveHourDelay, + Error: &Error{Code: "rate_limit_error", Message: "quota exhausted", HTTPStatus: http.StatusTooManyRequests}, + }) + + for _, modelID := range []string{allowedWindowModel, plainRPMModel} { + _, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: modelID}, cliproxyexecutor.Options{}) + var cdErr *modelCooldownError + if !errors.As(errExec, &cdErr) { + t.Fatalf("%s Execute() error = %T (%v), want *modelCooldownError", modelID, errExec, errExec) + } + hdr := cdErr.Headers() + if got := hdr.Get("Retry-After"); got == "" { + t.Fatalf("%s Retry-After = empty, want positive seconds", modelID) + } + for key := range hdr { + if strings.HasPrefix(strings.ToLower(key), "anthropic-ratelimit-unified-") { + t.Fatalf("%s ordinary rate-limit cooldown leaked unified header %s=%q", modelID, key, hdr.Get(key)) + } + } + } + + _, errModelQuota := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: modelQuotaModel}, cliproxyexecutor.Options{}) + var modelQuotaCooldown *modelCooldownError + if !errors.As(errModelQuota, &modelQuotaCooldown) { + t.Fatalf("modelQuota Execute() error = %T (%v), want *modelCooldownError", errModelQuota, errModelQuota) + } + if got := modelQuotaCooldown.Headers().Get("Anthropic-Ratelimit-Unified-Status"); got != "rejected" { + t.Fatalf("modelQuota Anthropic-Ratelimit-Unified-Status = %q, want rejected", got) + } + if got := modelQuotaCooldown.Headers().Get("Anthropic-Ratelimit-Unified-Representative-Claim"); got != "five_hour" { + t.Fatalf("modelQuota Representative-Claim = %q, want five_hour", got) + } + }) + } + }) + + t.Run("retains unified headers when late success refreshes passive signals during active cooldown", func(t *testing.T) { + for _, useAffinity := range []bool{true, false} { + pathName := "scheduler" + if useAffinity { + pathName = "session_affinity" + } + t.Run(pathName, func(t *testing.T) { + var selector Selector + if useAffinity { + selector = NewSessionAffinitySelector(&RoundRobinSelector{}) + } + manager := NewManager(nil, selector, nil) + manager.RegisterExecutor(&mockCustomErrorExecutor{identifier: "claude"}) + + credModel := "issue-7417-late-cred-" + pathName + credAuth := "issue-7417-late-cred-auth-" + pathName + restoredModel := "issue-7417-late-restored-" + pathName + restoredAuth := "issue-7417-late-restored-auth-" + pathName + modelScopedModel := "issue-7417-late-model-" + pathName + modelScopedAuth := "issue-7417-late-model-auth-" + pathName + + registry.GetGlobalRegistry().RegisterClient(credAuth, "claude", []*registry.ModelInfo{{ID: credModel}}) + registry.GetGlobalRegistry().RegisterClient(restoredAuth, "claude", []*registry.ModelInfo{{ID: restoredModel}}) + registry.GetGlobalRegistry().RegisterClient(modelScopedAuth, "claude", []*registry.ModelInfo{{ID: modelScopedModel}}) + t.Cleanup(func() { + registry.GetGlobalRegistry().UnregisterClient(credAuth) + registry.GetGlobalRegistry().UnregisterClient(restoredAuth) + registry.GetGlobalRegistry().UnregisterClient(modelScopedAuth) + }) + + if _, errRegister := manager.Register(context.Background(), &Auth{ + ID: credAuth, + Provider: "claude", + Status: StatusActive, + }); errRegister != nil { + t.Fatalf("register credAuth: %v", errRegister) + } + recoverAt := time.Now().Add(2 * time.Hour) + if _, errRegister := manager.Register(context.Background(), &Auth{ + ID: restoredAuth, + Provider: "claude", + Status: StatusError, + Unavailable: true, + NextRetryAfter: recoverAt, + Quota: QuotaState{ + Exceeded: true, + Reason: "credential_quota", + NextRecoverAt: recoverAt, + }, + }); errRegister != nil { + t.Fatalf("register restoredAuth: %v", errRegister) + } + if _, errRegister := manager.Register(context.Background(), &Auth{ + ID: modelScopedAuth, + Provider: "claude", + Status: StatusActive, + }); errRegister != nil { + t.Fatalf("register modelScopedAuth: %v", errRegister) + } + + delay := 2 * time.Hour + rejectedCtx := internallogging.WithResponseHeadersHolder(context.Background()) + internallogging.SetResponseHeaders(rejectedCtx, http.Header{ + "Anthropic-Ratelimit-Unified-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-7d-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-5h-Status": {"allowed"}, + "Anthropic-Ratelimit-Unified-Representative-Claim": {"seven_day"}, + "Anthropic-Ratelimit-Unified-Reset": {strconv.FormatInt(time.Now().Add(delay).Unix(), 10)}, + }) + + // Step 1: Record subscription 429 on credAuth and modelScopedAuth. + staleSeq := manager.nextExecutionSeq() + failureSeq := manager.nextExecutionSeq() + manager.MarkResult(rejectedCtx, Result{ + AuthID: credAuth, + Provider: "claude", + Model: credModel, + Success: false, + CredentialScope: true, + RetryAfter: &delay, + ExecutionSeq: failureSeq, + Error: &Error{Code: ErrorCodeCredentialQuota, Message: "rate_limit_error", HTTPStatus: http.StatusTooManyRequests}, + }) + manager.MarkResult(rejectedCtx, Result{ + AuthID: modelScopedAuth, + Provider: "claude", + Model: modelScopedModel, + Success: false, + CredentialScope: false, + RetryAfter: &delay, + ExecutionSeq: failureSeq, + Error: &Error{Code: "rate_limit_error", Message: "rate_limit_error", HTTPStatus: http.StatusTooManyRequests}, + }) + + // Step 2: Earlier in-flight requests complete with 200 OK and allowed unified headers. + allowedCtx := internallogging.WithResponseHeadersHolder(context.Background()) + internallogging.SetResponseHeaders(allowedCtx, http.Header{ + "Anthropic-Ratelimit-Unified-Status": {"allowed"}, + "Anthropic-Ratelimit-Unified-5h-Status": {"allowed"}, + "Anthropic-Ratelimit-Unified-7d-Status": {"allowed"}, + }) + manager.MarkResult(allowedCtx, Result{ + AuthID: credAuth, + Provider: "claude", + Model: credModel, + Success: true, + ExecutionSeq: staleSeq, + }) + manager.MarkResult(allowedCtx, Result{ + AuthID: restoredAuth, + Provider: "claude", + Model: restoredModel, + Success: true, + ExecutionSeq: staleSeq, + }) + manager.MarkResult(allowedCtx, Result{ + AuthID: modelScopedAuth, + Provider: "claude", + Model: modelScopedModel, + Success: true, + ExecutionSeq: staleSeq, + }) + + // Step 3: Subsequent requests during active cooldown still emit unified rate-limit headers. + for _, tc := range []struct { + model string + wantClaim string + }{ + {model: credModel, wantClaim: "seven_day"}, + {model: restoredModel, wantClaim: "five_hour"}, + {model: modelScopedModel, wantClaim: "seven_day"}, + } { + _, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: tc.model}, cliproxyexecutor.Options{}) + var cdErr *modelCooldownError + if !errors.As(errExec, &cdErr) { + t.Fatalf("%s Execute() error = %T (%v), want *modelCooldownError", tc.model, errExec, errExec) + } + hdr := cdErr.Headers() + if got := hdr.Get("Anthropic-Ratelimit-Unified-Status"); got != "rejected" { + t.Fatalf("%s Anthropic-Ratelimit-Unified-Status = %q, want rejected", tc.model, got) + } + if got := hdr.Get("Anthropic-Ratelimit-Unified-Representative-Claim"); got != tc.wantClaim { + t.Fatalf("%s Representative-Claim = %q, want %s", tc.model, got, tc.wantClaim) + } + } + }) + } + }) + + t.Run("does not emit stale unified headers when 503 follows expired model quota cooldown", func(t *testing.T) { + for _, useAffinity := range []bool{true, false} { + pathName := "scheduler" + if useAffinity { + pathName = "session_affinity" + } + t.Run(pathName, func(t *testing.T) { + var selector Selector + if useAffinity { + selector = NewSessionAffinitySelector(&RoundRobinSelector{}) + } + manager := NewManager(nil, selector, nil) + manager.RegisterExecutor(&mockCustomErrorExecutor{identifier: "claude"}) + + modelID := "issue-7417-expired-503-" + pathName + authID := "issue-7417-expired-503-auth-" + pathName + registry.GetGlobalRegistry().RegisterClient(authID, "claude", []*registry.ModelInfo{{ID: modelID}}) + t.Cleanup(func() { + registry.GetGlobalRegistry().UnregisterClient(authID) + }) + + if _, errRegister := manager.Register(context.Background(), &Auth{ + ID: authID, + Provider: "claude", + Status: StatusActive, + }); errRegister != nil { + t.Fatalf("register auth: %v", errRegister) + } + + // 1. Model-scoped unified 429 records rejected signals. + quotaDelay := time.Hour + rejectedCtx := internallogging.WithResponseHeadersHolder(context.Background()) + internallogging.SetResponseHeaders(rejectedCtx, http.Header{ + "Anthropic-Ratelimit-Unified-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-5h-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-Representative-Claim": {"five_hour"}, + "Anthropic-Ratelimit-Unified-Reset": {strconv.FormatInt(time.Now().Add(quotaDelay).Unix(), 10)}, + }) + manager.MarkResult(rejectedCtx, Result{ + AuthID: authID, + Provider: "claude", + Model: modelID, + Success: false, + CredentialScope: false, + RetryAfter: "aDelay, + Error: &Error{Code: "rate_limit_error", Message: "quota", HTTPStatus: http.StatusTooManyRequests}, + }) + + // 2. Simulate natural expiry of the quota deadline without wall-clock sleep. + past := time.Now().Add(-time.Minute) + if _, okModify, errModify := manager.Modify(context.Background(), authID, 0, func(a *Auth) error { + a.NextRetryAfter = past + if state := a.ModelStates[modelID]; state != nil { + state.NextRetryAfter = past + state.Quota.NextRecoverAt = past + } + a.Quota.NextRecoverAt = past + return nil + }); errModify != nil || !okModify { + t.Fatalf("Modify() ok=%v err=%v", okModify, errModify) + } + + // 3. Subsequent request fails with 503 Service Unavailable and Retry-After. + serverDelay := 45 * time.Second + manager.MarkResult(context.Background(), Result{ + AuthID: authID, + Provider: "claude", + Model: modelID, + Success: false, + RetryAfter: &serverDelay, + Error: &Error{Code: "server_error", Message: "overloaded", HTTPStatus: http.StatusServiceUnavailable}, + }) + + // 4. Subsequent selection error must not advertise unified subscription exhaustion. + _, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: modelID}, cliproxyexecutor.Options{}) + if errExec == nil { + t.Fatal("expected unavailable/cooldown error after 503, got nil") + } + if carrier, ok := errExec.(interface{ Headers() http.Header }); ok && carrier != nil { + for key := range carrier.Headers() { + if strings.HasPrefix(strings.ToLower(key), "anthropic-ratelimit-unified-") { + t.Fatalf("%s post-expiry 503 leaked unified header %s=%q", pathName, key, carrier.Headers().Get(key)) + } + } + } + }) + } + }) + + t.Run("persists and restores model-scoped unified cooldown headers across restart", func(t *testing.T) { + for _, useAffinity := range []bool{true, false} { + pathName := "scheduler" + if useAffinity { + pathName = "session_affinity" + } + t.Run(pathName, func(t *testing.T) { + store := NewFileCooldownStateStore(t.TempDir()) + modelID := "issue-7417-restart-model-" + pathName + authID := "issue-7417-restart-auth-" + pathName + registry.GetGlobalRegistry().RegisterClient(authID, "claude", []*registry.ModelInfo{{ID: modelID}}) + t.Cleanup(func() { + registry.GetGlobalRegistry().UnregisterClient(authID) + }) + + writerMgr := NewManager(nil, nil, nil) + writerMgr.SetCooldownStateStore(store) + if _, errRegister := writerMgr.Register(context.Background(), &Auth{ + ID: authID, + Provider: "claude", + Status: StatusActive, + }); errRegister != nil { + t.Fatalf("writer Register: %v", errRegister) + } + + delay := 2 * time.Hour + rejectedCtx := internallogging.WithResponseHeadersHolder(context.Background()) + internallogging.SetResponseHeaders(rejectedCtx, http.Header{ + "Anthropic-Ratelimit-Unified-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-7d-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-5h-Status": {"allowed"}, + "Anthropic-Ratelimit-Unified-Representative-Claim": {"seven_day"}, + "Anthropic-Ratelimit-Unified-Reset": {strconv.FormatInt(time.Now().Add(delay).Unix(), 10)}, + }) + writerMgr.MarkResult(rejectedCtx, Result{ + AuthID: authID, + Provider: "claude", + Model: modelID, + Success: false, + CredentialScope: false, + RetryAfter: &delay, + Error: &Error{Code: "rate_limit_error", Message: "quota", HTTPStatus: http.StatusTooManyRequests}, + }) + writerMgr.PersistCooldownStates(context.Background()) + + var selector Selector + if useAffinity { + selector = NewSessionAffinitySelector(&RoundRobinSelector{}) + } + restoredMgr := NewManager(nil, selector, nil) + restoredMgr.RegisterExecutor(&mockCustomErrorExecutor{identifier: "claude"}) + restoredMgr.SetCooldownStateStore(store) + if _, errRegister := restoredMgr.Register(context.Background(), &Auth{ + ID: authID, + Provider: "claude", + Status: StatusActive, + }); errRegister != nil { + t.Fatalf("restored Register: %v", errRegister) + } + if errRestore := restoredMgr.RestoreCooldownStates(context.Background()); errRestore != nil { + t.Fatalf("RestoreCooldownStates: %v", errRestore) + } + + _, errExec := restoredMgr.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: modelID}, cliproxyexecutor.Options{}) + var cdErr *modelCooldownError + if !errors.As(errExec, &cdErr) { + t.Fatalf("%s Execute() error = %T (%v), want *modelCooldownError", pathName, errExec, errExec) + } + hdr := cdErr.Headers() + if got := hdr.Get("Anthropic-Ratelimit-Unified-Status"); got != "rejected" { + t.Fatalf("%s restored Anthropic-Ratelimit-Unified-Status = %q, want rejected", pathName, got) + } + if got := hdr.Get("Anthropic-Ratelimit-Unified-Representative-Claim"); got != "seven_day" { + t.Fatalf("%s restored Representative-Claim = %q, want seven_day", pathName, got) + } + if got := hdr.Get("Anthropic-Ratelimit-Unified-7d-Status"); got != "rejected" { + t.Fatalf("%s restored 7d-Status = %q, want rejected", pathName, got) + } + if got := hdr.Get("Anthropic-Ratelimit-Unified-5h-Status"); got != "allowed" { + t.Fatalf("%s restored 5h-Status = %q, want allowed", pathName, got) + } + }) + } + }) + + t.Run("keeps longer cooldown window signals when shorter out-of-order 429 completes later", func(t *testing.T) { + for _, useAffinity := range []bool{true, false} { + pathName := "scheduler" + if useAffinity { + pathName = "session_affinity" + } + t.Run(pathName, func(t *testing.T) { + var selector Selector + if useAffinity { + selector = NewSessionAffinitySelector(&RoundRobinSelector{}) + } + manager := NewManager(nil, selector, nil) + manager.RegisterExecutor(&mockCustomErrorExecutor{identifier: "claude"}) + + credModelA := "issue-7417-ooo-cred-a-" + pathName + credModelB := "issue-7417-ooo-cred-b-" + pathName + credAuth := "issue-7417-ooo-cred-auth-" + pathName + modelScopedModel := "issue-7417-ooo-model-" + pathName + modelScopedAuth := "issue-7417-ooo-model-auth-" + pathName + rpmFirstModel := "issue-7417-ooo-rpm-" + pathName + rpmFirstAuth := "issue-7417-ooo-rpm-auth-" + pathName + + registry.GetGlobalRegistry().RegisterClient(credAuth, "claude", []*registry.ModelInfo{{ID: credModelA}, {ID: credModelB}}) + registry.GetGlobalRegistry().RegisterClient(modelScopedAuth, "claude", []*registry.ModelInfo{{ID: modelScopedModel}}) + registry.GetGlobalRegistry().RegisterClient(rpmFirstAuth, "claude", []*registry.ModelInfo{{ID: rpmFirstModel}}) + t.Cleanup(func() { + registry.GetGlobalRegistry().UnregisterClient(credAuth) + registry.GetGlobalRegistry().UnregisterClient(modelScopedAuth) + registry.GetGlobalRegistry().UnregisterClient(rpmFirstAuth) + }) + + for _, id := range []string{credAuth, modelScopedAuth, rpmFirstAuth} { + if _, errRegister := manager.Register(context.Background(), &Auth{ + ID: id, + Provider: "claude", + Status: StatusActive, + }); errRegister != nil { + t.Fatalf("register %s: %v", id, errRegister) + } + } + + longDelay := 2 * time.Hour + shortDelay := time.Hour + sevenDayCtx := internallogging.WithResponseHeadersHolder(context.Background()) + internallogging.SetResponseHeaders(sevenDayCtx, http.Header{ + "Anthropic-Ratelimit-Unified-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-7d-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-5h-Status": {"allowed"}, + "Anthropic-Ratelimit-Unified-Representative-Claim": {"seven_day"}, + "Anthropic-Ratelimit-Unified-Reset": {strconv.FormatInt(time.Now().Add(longDelay).Unix(), 10)}, + }) + fiveHourCtx := internallogging.WithResponseHeadersHolder(context.Background()) + internallogging.SetResponseHeaders(fiveHourCtx, http.Header{ + "Anthropic-Ratelimit-Unified-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-5h-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-7d-Status": {"allowed"}, + "Anthropic-Ratelimit-Unified-Representative-Claim": {"five_hour"}, + "Anthropic-Ratelimit-Unified-Reset": {strconv.FormatInt(time.Now().Add(shortDelay).Unix(), 10)}, + }) + + // Credential-scoped: 2h seven_day on model A, then shorter 1h five_hour on model B. + manager.MarkResult(sevenDayCtx, Result{ + AuthID: credAuth, + Provider: "claude", + Model: credModelA, + Success: false, + CredentialScope: true, + RetryAfter: &longDelay, + Error: &Error{Code: ErrorCodeCredentialQuota, Message: "7d quota", HTTPStatus: http.StatusTooManyRequests}, + }) + manager.MarkResult(fiveHourCtx, Result{ + AuthID: credAuth, + Provider: "claude", + Model: credModelB, + Success: false, + CredentialScope: true, + RetryAfter: &shortDelay, + Error: &Error{Code: ErrorCodeCredentialQuota, Message: "5h quota", HTTPStatus: http.StatusTooManyRequests}, + }) + + // Model-scoped: 2h seven_day, then shorter 1h five_hour. + manager.MarkResult(sevenDayCtx, Result{ + AuthID: modelScopedAuth, + Provider: "claude", + Model: modelScopedModel, + Success: false, + CredentialScope: false, + RetryAfter: &longDelay, + Error: &Error{Code: "rate_limit_error", Message: "7d quota", HTTPStatus: http.StatusTooManyRequests}, + }) + manager.MarkResult(fiveHourCtx, Result{ + AuthID: modelScopedAuth, + Provider: "claude", + Model: modelScopedModel, + Success: false, + CredentialScope: false, + RetryAfter: &shortDelay, + Error: &Error{Code: "rate_limit_error", Message: "5h quota", HTTPStatus: http.StatusTooManyRequests}, + }) + + // Longer ordinary 10m RPM 429 followed by shorter 1m unified 429: governing deadline stays ordinary RPM. + rpmLong := 10 * time.Minute + unifiedShort := time.Minute + rpmCtx := internallogging.WithResponseHeadersHolder(context.Background()) + internallogging.SetResponseHeaders(rpmCtx, http.Header{"Retry-After": {"600"}}) + manager.MarkResult(rpmCtx, Result{ + AuthID: rpmFirstAuth, + Provider: "claude", + Model: rpmFirstModel, + Success: false, + CredentialScope: false, + RetryAfter: &rpmLong, + Error: &Error{Code: "rate_limit_error", Message: "rpm", HTTPStatus: http.StatusTooManyRequests}, + }) + manager.MarkResult(fiveHourCtx, Result{ + AuthID: rpmFirstAuth, + Provider: "claude", + Model: rpmFirstModel, + Success: false, + CredentialScope: false, + RetryAfter: &unifiedShort, + Error: &Error{Code: "rate_limit_error", Message: "5h", HTTPStatus: http.StatusTooManyRequests}, + }) + + for _, m := range []string{credModelA, credModelB, modelScopedModel} { + _, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: m}, cliproxyexecutor.Options{}) + var cdErr *modelCooldownError + if !errors.As(errExec, &cdErr) { + t.Fatalf("%s Execute() error = %T (%v), want *modelCooldownError", m, errExec, errExec) + } + if got := cdErr.Headers().Get("Anthropic-Ratelimit-Unified-Representative-Claim"); got != "seven_day" { + t.Fatalf("%s Representative-Claim = %q, want seven_day (from governing 2h deadline)", m, got) + } + if got := cdErr.Headers().Get("Anthropic-Ratelimit-Unified-7d-Status"); got != "rejected" { + t.Fatalf("%s 7d-Status = %q, want rejected", m, got) + } + } + + _, errRPM := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: rpmFirstModel}, cliproxyexecutor.Options{}) + var rpmCdErr *modelCooldownError + if !errors.As(errRPM, &rpmCdErr) { + t.Fatalf("%s Execute() error = %T (%v), want *modelCooldownError", rpmFirstModel, errRPM, errRPM) + } + if got := rpmCdErr.Headers().Get("Anthropic-Ratelimit-Unified-Status"); got != "" { + t.Fatalf("%s leaked unified status %q while 10m ordinary RPM deadline still governs", rpmFirstModel, got) + } + }) + } + }) + + t.Run("reverts to independent model layer after propagated credential quota expires", func(t *testing.T) { + for _, useAffinity := range []bool{true, false} { + pathName := "scheduler" + if useAffinity { + pathName = "session_affinity" + } + t.Run(pathName, func(t *testing.T) { + var selector Selector + if useAffinity { + selector = NewSessionAffinitySelector(&RoundRobinSelector{}) + } + manager := NewManager(nil, selector, nil) + manager.RegisterExecutor(&mockCustomErrorExecutor{identifier: "claude"}) + + modelA := "issue-7417-shadow-a-" + pathName + modelB := "issue-7417-shadow-b-" + pathName + authID := "issue-7417-shadow-auth-" + pathName + registry.GetGlobalRegistry().RegisterClient(authID, "claude", []*registry.ModelInfo{{ID: modelA}, {ID: modelB}}) + t.Cleanup(func() { + registry.GetGlobalRegistry().UnregisterClient(authID) + }) + + if _, errRegister := manager.Register(context.Background(), &Auth{ + ID: authID, + Provider: "claude", + Status: StatusActive, + }); errRegister != nil { + t.Fatalf("register auth: %v", errRegister) + } + + // 1. Model A enters a 10m ordinary RPM cooldown (no unified rejection headers). + rpmDelay := 10 * time.Minute + rpmCtx := internallogging.WithResponseHeadersHolder(context.Background()) + internallogging.SetResponseHeaders(rpmCtx, http.Header{"Retry-After": {"600"}}) + manager.MarkResult(rpmCtx, Result{ + AuthID: authID, + Provider: "claude", + Model: modelA, + Success: false, + CredentialScope: false, + RetryAfter: &rpmDelay, + Error: &Error{Code: "rate_limit_error", Message: "rpm", HTTPStatus: http.StatusTooManyRequests}, + }) + + // 2. Model B triggers a shorter 1m credential-wide 5h quota cooldown. + credDelay := time.Minute + fiveHourCtx := internallogging.WithResponseHeadersHolder(context.Background()) + internallogging.SetResponseHeaders(fiveHourCtx, http.Header{ + "Anthropic-Ratelimit-Unified-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-5h-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-Representative-Claim": {"five_hour"}, + "Anthropic-Ratelimit-Unified-Reset": {strconv.FormatInt(time.Now().Add(credDelay).Unix(), 10)}, + }) + manager.MarkResult(fiveHourCtx, Result{ + AuthID: authID, + Provider: "claude", + Model: modelB, + Success: false, + CredentialScope: true, + RetryAfter: &credDelay, + Error: &Error{Code: ErrorCodeCredentialQuota, Message: "5h quota", HTTPStatus: http.StatusTooManyRequests}, + }) + + // 3. Expire the 1m credential-wide quota deterministically while Model A's 10m RPM block remains live. + past := time.Now().Add(-time.Second) + manager.mu.Lock() + curAuth := manager.auths[authID] + curAuth.Quota.NextRecoverAt = past + curAuth.NextRetryAfter = past + if sb := curAuth.ModelStates[modelB]; sb != nil { + sb.NextRetryAfter = past + sb.Quota.NextRecoverAt = past + } + snapshot := curAuth.Clone() + manager.mu.Unlock() + if manager.scheduler != nil { + manager.scheduler.upsertAuth(snapshot) + } + + // 4. Model A is still cooling down from its independent 10m RPM error and must not emit 5h unified headers. + _, errExecA := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: modelA}, cliproxyexecutor.Options{}) + var cdErrA *modelCooldownError + if !errors.As(errExecA, &cdErrA) { + t.Fatalf("%s modelA Execute() error = %T (%v), want *modelCooldownError", pathName, errExecA, errExecA) + } + for key := range cdErrA.Headers() { + if strings.HasPrefix(strings.ToLower(key), "anthropic-ratelimit-unified-") { + t.Fatalf("%s modelA leaked expired credential-quota header %s=%q", pathName, key, cdErrA.Headers().Get(key)) + } + } + }) + } + }) + + t.Run("matches longer independent model quota signals and deadline beneath shorter credential quota", func(t *testing.T) { + for _, useAffinity := range []bool{true, false} { + pathName := "scheduler" + if useAffinity { + pathName = "session_affinity" + } + t.Run(pathName, func(t *testing.T) { + var selector Selector + if useAffinity { + selector = NewSessionAffinitySelector(&RoundRobinSelector{}) + } + manager := NewManager(nil, selector, nil) + manager.RegisterExecutor(&mockCustomErrorExecutor{identifier: "claude"}) + + modelA := "issue-7417-dual-quota-a-" + pathName + modelB := "issue-7417-dual-quota-b-" + pathName + authID := "issue-7417-dual-quota-auth-" + pathName + registry.GetGlobalRegistry().RegisterClient(authID, "claude", []*registry.ModelInfo{{ID: modelA}, {ID: modelB}}) + t.Cleanup(func() { + registry.GetGlobalRegistry().UnregisterClient(authID) + }) + + if _, errRegister := manager.Register(context.Background(), &Auth{ + ID: authID, + Provider: "claude", + Status: StatusActive, + }); errRegister != nil { + t.Fatalf("register auth: %v", errRegister) + } + + // 1. Model A enters a 2h independent model-scoped seven_day quota cooldown. + twoHours := 2 * time.Hour + sevenDayCtx := internallogging.WithResponseHeadersHolder(context.Background()) + internallogging.SetResponseHeaders(sevenDayCtx, http.Header{ + "Anthropic-Ratelimit-Unified-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-7d-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-5h-Status": {"allowed"}, + "Anthropic-Ratelimit-Unified-Representative-Claim": {"seven_day"}, + "Anthropic-Ratelimit-Unified-Reset": {strconv.FormatInt(time.Now().Add(twoHours).Unix(), 10)}, + }) + manager.MarkResult(sevenDayCtx, Result{ + AuthID: authID, + Provider: "claude", + Model: modelA, + Success: false, + CredentialScope: false, + RetryAfter: &twoHours, + Error: &Error{Code: "rate_limit_error", Message: "7d model quota", HTTPStatus: http.StatusTooManyRequests}, + }) + + // 2. Model B triggers a shorter 1h credential-scoped five_hour quota cooldown. + oneHour := time.Hour + fiveHourCtx := internallogging.WithResponseHeadersHolder(context.Background()) + internallogging.SetResponseHeaders(fiveHourCtx, http.Header{ + "Anthropic-Ratelimit-Unified-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-5h-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-7d-Status": {"allowed"}, + "Anthropic-Ratelimit-Unified-Representative-Claim": {"five_hour"}, + "Anthropic-Ratelimit-Unified-Reset": {strconv.FormatInt(time.Now().Add(oneHour).Unix(), 10)}, + }) + manager.MarkResult(fiveHourCtx, Result{ + AuthID: authID, + Provider: "claude", + Model: modelB, + Success: false, + CredentialScope: true, + RetryAfter: &oneHour, + Error: &Error{Code: ErrorCodeCredentialQuota, Message: "5h cred quota", HTTPStatus: http.StatusTooManyRequests}, + }) + + // Model A must report seven_day AND ~2h Retry-After / reset (not 1h). + _, errExecA := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: modelA}, cliproxyexecutor.Options{}) + var cdErrA *modelCooldownError + if !errors.As(errExecA, &cdErrA) { + t.Fatalf("%s modelA Execute() error = %T (%v), want *modelCooldownError", pathName, errExecA, errExecA) + } + hdrA := cdErrA.Headers() + if got := hdrA.Get("Anthropic-Ratelimit-Unified-Representative-Claim"); got != "seven_day" { + t.Fatalf("%s modelA claim = %q, want seven_day", pathName, got) + } + retrySecA, errSecA := strconv.ParseInt(hdrA.Get("Retry-After"), 10, 64) + if errSecA != nil || retrySecA < 7100 { + t.Fatalf("%s modelA Retry-After = %q, want ~7200s (>= 7100s) matching 2h independent deadline", pathName, hdrA.Get("Retry-After")) + } + + // Model B must report five_hour AND ~1h Retry-After. + _, errExecB := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: modelB}, cliproxyexecutor.Options{}) + var cdErrB *modelCooldownError + if !errors.As(errExecB, &cdErrB) { + t.Fatalf("%s modelB Execute() error = %T (%v), want *modelCooldownError", pathName, errExecB, errExecB) + } + hdrB := cdErrB.Headers() + if got := hdrB.Get("Anthropic-Ratelimit-Unified-Representative-Claim"); got != "five_hour" { + t.Fatalf("%s modelB claim = %q, want five_hour", pathName, got) + } + retrySecB, errSecB := strconv.ParseInt(hdrB.Get("Retry-After"), 10, 64) + if errSecB != nil || retrySecB > 3660 || retrySecB < 3500 { + t.Fatalf("%s modelB Retry-After = %q, want ~3600s", pathName, hdrB.Get("Retry-After")) + } + }) + } + }) + + t.Run("keeps active cooldown signals when hintless RetryAfter nil 429 reuses existing deadline", func(t *testing.T) { + for _, useAffinity := range []bool{true, false} { + pathName := "scheduler" + if useAffinity { + pathName = "session_affinity" + } + t.Run(pathName, func(t *testing.T) { + var selector Selector + if useAffinity { + selector = NewSessionAffinitySelector(&RoundRobinSelector{}) + } + manager := NewManager(nil, selector, nil) + manager.RegisterExecutor(&mockCustomErrorExecutor{identifier: "claude"}) + + credModel := "issue-7417-nilhint-cred-" + pathName + credAuth := "issue-7417-nilhint-cred-auth-" + pathName + modelScopedModel := "issue-7417-nilhint-model-" + pathName + modelScopedAuth := "issue-7417-nilhint-model-auth-" + pathName + authOnlyModel := "issue-7417-nilhint-authonly-" + pathName + authOnlyID := "issue-7417-nilhint-authonly-auth-" + pathName + + registry.GetGlobalRegistry().RegisterClient(credAuth, "claude", []*registry.ModelInfo{{ID: credModel}}) + registry.GetGlobalRegistry().RegisterClient(modelScopedAuth, "claude", []*registry.ModelInfo{{ID: modelScopedModel}}) + registry.GetGlobalRegistry().RegisterClient(authOnlyID, "claude", []*registry.ModelInfo{{ID: authOnlyModel}}) + t.Cleanup(func() { + registry.GetGlobalRegistry().UnregisterClient(credAuth) + registry.GetGlobalRegistry().UnregisterClient(modelScopedAuth) + registry.GetGlobalRegistry().UnregisterClient(authOnlyID) + }) + + for _, id := range []string{credAuth, modelScopedAuth, authOnlyID} { + if _, errRegister := manager.Register(context.Background(), &Auth{ + ID: id, + Provider: "claude", + Status: StatusActive, + }); errRegister != nil { + t.Fatalf("register %s: %v", id, errRegister) + } + } + + twoHours := 2 * time.Hour + sevenDayCtx := internallogging.WithResponseHeadersHolder(context.Background()) + internallogging.SetResponseHeaders(sevenDayCtx, http.Header{ + "Anthropic-Ratelimit-Unified-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-7d-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-5h-Status": {"allowed"}, + "Anthropic-Ratelimit-Unified-Representative-Claim": {"seven_day"}, + "Anthropic-Ratelimit-Unified-Reset": {strconv.FormatInt(time.Now().Add(twoHours).Unix(), 10)}, + }) + fiveHourNoHintCtx := internallogging.WithResponseHeadersHolder(context.Background()) + internallogging.SetResponseHeaders(fiveHourNoHintCtx, http.Header{ + "Anthropic-Ratelimit-Unified-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-5h-Status": {"rejected"}, + "Anthropic-Ratelimit-Unified-7d-Status": {"allowed"}, + "Anthropic-Ratelimit-Unified-Representative-Claim": {"five_hour"}, + }) + + manager.MarkResult(sevenDayCtx, Result{ + AuthID: credAuth, + Provider: "claude", + Model: credModel, + Success: false, + CredentialScope: true, + RetryAfter: &twoHours, + Error: &Error{Code: ErrorCodeCredentialQuota, Message: "7d", HTTPStatus: http.StatusTooManyRequests}, + }) + manager.MarkResult(fiveHourNoHintCtx, Result{ + AuthID: credAuth, + Provider: "claude", + Model: credModel, + Success: false, + CredentialScope: true, + RetryAfter: nil, + Error: &Error{Code: ErrorCodeCredentialQuota, Message: "5h no hint", HTTPStatus: http.StatusTooManyRequests}, + }) + + manager.MarkResult(sevenDayCtx, Result{ + AuthID: modelScopedAuth, + Provider: "claude", + Model: modelScopedModel, + Success: false, + CredentialScope: false, + RetryAfter: &twoHours, + Error: &Error{Code: "rate_limit_error", Message: "7d", HTTPStatus: http.StatusTooManyRequests}, + }) + manager.MarkResult(fiveHourNoHintCtx, Result{ + AuthID: modelScopedAuth, + Provider: "claude", + Model: modelScopedModel, + Success: false, + CredentialScope: false, + RetryAfter: nil, + Error: &Error{Code: "rate_limit_error", Message: "5h no hint", HTTPStatus: http.StatusTooManyRequests}, + }) + + manager.MarkResult(sevenDayCtx, Result{ + AuthID: authOnlyID, + Provider: "claude", + Success: false, + RetryAfter: &twoHours, + Error: &Error{Code: "rate_limit_error", Message: "7d", HTTPStatus: http.StatusTooManyRequests}, + }) + manager.MarkResult(fiveHourNoHintCtx, Result{ + AuthID: authOnlyID, + Provider: "claude", + Success: false, + RetryAfter: nil, + Error: &Error{Code: "rate_limit_error", Message: "5h no hint", HTTPStatus: http.StatusTooManyRequests}, + }) + + for _, m := range []string{credModel, modelScopedModel, authOnlyModel} { + _, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: m}, cliproxyexecutor.Options{}) + var cdErr *modelCooldownError + if !errors.As(errExec, &cdErr) { + t.Fatalf("%s Execute() error = %T (%v), want *modelCooldownError", m, errExec, errExec) + } + if got := cdErr.Headers().Get("Anthropic-Ratelimit-Unified-Representative-Claim"); got != "seven_day" { + t.Fatalf("%s Representative-Claim = %q, want seven_day", m, got) + } + } + }) + } + }) +} diff --git a/sdk/cliproxy/auth/cooldown_state.go b/sdk/cliproxy/auth/cooldown_state.go index 28c9c91f04..c3b1bafac8 100644 --- a/sdk/cliproxy/auth/cooldown_state.go +++ b/sdk/cliproxy/auth/cooldown_state.go @@ -34,6 +34,11 @@ type CooldownStateRecord struct { // CredentialQuotaPropagated preserves wrap provenance for models that // carry no independent layer (e.g. the reporting model itself). CredentialQuotaPropagated bool `json:"credential_quota_propagated,omitempty"` + // CooldownSignals persists bounded rejected quota headers captured when + // this cooldown was recorded without touching passive Quota.Signals watermarks. + CooldownSignals map[string]string `json:"cooldown_signals,omitempty"` + // IndependentCooldownSignals persists CooldownSignals for IndependentState. + IndependentCooldownSignals map[string]string `json:"independent_cooldown_signals,omitempty"` } // CooldownStateStore persists runtime cooldown state independently from auth tokens. diff --git a/sdk/cliproxy/auth/home_concurrency.go b/sdk/cliproxy/auth/home_concurrency.go index 15d5c291d5..8a3f47a4db 100644 --- a/sdk/cliproxy/auth/home_concurrency.go +++ b/sdk/cliproxy/auth/home_concurrency.go @@ -310,7 +310,7 @@ func SafeResponseHeaders(err error) http.Header { } var modelCooldown *modelCooldownError if errors.As(err, &modelCooldown) && modelCooldown != nil { - return modelCooldown.Headers() + return modelCooldown.SafeResponseHeaders() } var unavailable *authUnavailableError if errors.As(err, &unavailable) && unavailable != nil { diff --git a/sdk/cliproxy/auth/quota_signals.go b/sdk/cliproxy/auth/quota_signals.go index fea368e531..071158c622 100644 --- a/sdk/cliproxy/auth/quota_signals.go +++ b/sdk/cliproxy/auth/quota_signals.go @@ -1,6 +1,7 @@ package auth import ( + "maps" "net/http" "sort" "strings" @@ -86,6 +87,32 @@ func applyCooldownFields(dst *QuotaState, cooldown QuotaState) { dst.Reason = cooldown.Reason dst.NextRecoverAt = cooldown.NextRecoverAt dst.BackoffLevel = cooldown.BackoffLevel + dst.cooldownSignals = maps.Clone(cooldown.cooldownSignals) + dst.cooldownEvaluated = cooldown.Exceeded || cooldown.cooldownEvaluated +} + +func rejectedClaudeSignalsForCooldown(provider string, headers http.Header) map[string]string { + if !strings.EqualFold(strings.TrimSpace(provider), "claude") { + return nil + } + signals := collectQuotaSignals("claude", headers) + if !hasRejectedClaudeUnifiedSignals(signals) { + return nil + } + return signals +} + +func sanitizePersistedClaudeCooldownSignals(raw map[string]string) map[string]string { + if len(raw) == 0 { + return nil + } + headers := make(http.Header, len(raw)) + for key, val := range raw { + if strings.HasPrefix(strings.ToLower(strings.TrimSpace(key)), "anthropic-ratelimit-unified-") { + headers.Set(key, val) + } + } + return rejectedClaudeSignalsForCooldown("claude", headers) } // collectQuotaSignals builds the bounded snapshot for a single response. diff --git a/sdk/cliproxy/auth/scheduler.go b/sdk/cliproxy/auth/scheduler.go index b62e332d14..87c15f6494 100644 --- a/sdk/cliproxy/auth/scheduler.go +++ b/sdk/cliproxy/auth/scheduler.go @@ -682,6 +682,11 @@ func (s *authScheduler) mixedUnavailableErrorLocked(providers []string, model st var latestAuthID string var latestAuthErr error + allClaude := true + var earliestClaudeTime time.Time + var earliestClaudeID string + var earliestClaudeSignals map[string]string + for _, providerKey := range providers { providerState := s.providers[providerKey] if providerState == nil { @@ -698,6 +703,18 @@ func (s *authScheduler) mixedUnavailableErrorLocked(providers []string, model st if !localEarliest.IsZero() && (earliest.IsZero() || localEarliest.Before(earliest)) { earliest = localEarliest } + if localTotal > 0 { + localClaude, localClaudeEarliest, localClaudeID, localSignals := shard.claudeCooldownSummaryLocked(providerKey, model, now, predicate) + if !localClaude { + allClaude = false + } else if allClaude && !localClaudeEarliest.IsZero() { + if earliestClaudeTime.IsZero() || localClaudeEarliest.Before(earliestClaudeTime) || (localClaudeEarliest.Equal(earliestClaudeTime) && localClaudeID < earliestClaudeID) { + earliestClaudeTime = localClaudeEarliest + earliestClaudeID = localClaudeID + earliestClaudeSignals = localSignals + } + } + } candModelErr, candModelTime, candModelAuthID, candAuthErr, candAuthTime, candAuthAuthID := shard.candidateErrorsLocked(model, predicate) if candModelErr != nil { if latestModelErr == nil || candModelTime.After(latestModelTime) || (candModelTime.Equal(latestModelTime) && candModelAuthID > latestModelAuthID) { @@ -730,7 +747,11 @@ func (s *authScheduler) mixedUnavailableErrorLocked(providers []string, model st if resetIn < 0 { resetIn = 0 } - return newModelCooldownErrorWithCause(model, "", resetIn, lastCandidateErr) + var claudeSignals map[string]string + if allClaude { + claudeSignals = earliestClaudeSignals + } + return newModelCooldownErrorForCandidates(model, "", resetIn, lastCandidateErr, allClaude, claudeSignals) } if unauthorizedCount == total { terminalCause := latestAuthErr @@ -1475,7 +1496,8 @@ func (m *modelScheduler) unavailableErrorLocked(provider, model string, predicat if resetIn < 0 { resetIn = 0 } - return newModelCooldownErrorWithCause(model, providerForError, resetIn, lastCandidateErr) + claudeCooldown, _, _, claudeSignals := m.claudeCooldownSummaryLocked(provider, model, now, predicate) + return newModelCooldownErrorForCandidates(model, providerForError, resetIn, lastCandidateErr, claudeCooldown, claudeSignals) } if unauthorizedCount == total { terminalCause := authErr @@ -1591,6 +1613,57 @@ func (m *modelScheduler) availabilitySummaryLocked(predicate func(*scheduledAuth return total, cooldownCount, unauthorizedCount, earliest } +func (m *modelScheduler) claudeCooldownSummaryLocked(fallbackProvider, model string, now time.Time, predicate func(*scheduledAuth) bool) (bool, time.Time, string, map[string]string) { + if m == nil { + return false, time.Time{}, "", nil + } + matched := 0 + var earliest time.Time + var earliestID string + var earliestAuth *Auth + var earliestSignals map[string]string + for _, entry := range m.entries { + if predicate != nil && !predicate(entry) { + continue + } + matched++ + if entry == nil || entry.auth == nil { + return false, time.Time{}, "", nil + } + entryProvider := "" + if entry.meta != nil && entry.meta.providerKey != "" { + entryProvider = entry.meta.providerKey + } else if resolved := executorKeyFromAuth(entry.auth); resolved != "" { + entryProvider = resolved + } else { + entryProvider = strings.ToLower(strings.TrimSpace(fallbackProvider)) + } + if entryProvider != "claude" { + return false, time.Time{}, "", nil + } + checkModel := model + if existingModelState(entry.auth, checkModel) == nil && m.modelKey != "" { + checkModel = m.modelKey + } + okClaude, signals := claudeUnifiedCooldownForAuth(entry.auth, checkModel, now) + if !okClaude { + return false, time.Time{}, "", nil + } + if entry.state == scheduledStateCooldown && !entry.nextRetryAt.IsZero() { + if earliestAuth == nil || entry.nextRetryAt.Before(earliest) || (entry.nextRetryAt.Equal(earliest) && entry.auth.ID < earliestID) { + earliest = entry.nextRetryAt + earliestID = entry.auth.ID + earliestAuth = entry.auth + earliestSignals = signals + } + } + } + if matched == 0 || earliestAuth == nil { + return false, time.Time{}, "", nil + } + return true, earliest, earliestID, earliestSignals +} + // rebuildIndexesLocked reconstructs ready and blocked views from the current entry map. func (m *modelScheduler) rebuildIndexesLocked() { cursorStates := make(map[int]readyBucketCursorState, len(m.readyByPriority)) diff --git a/sdk/cliproxy/auth/selector.go b/sdk/cliproxy/auth/selector.go index 24c76560e0..024681d9e5 100644 --- a/sdk/cliproxy/auth/selector.go +++ b/sdk/cliproxy/auth/selector.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "hash/fnv" + "maps" "math" "net/http" "regexp" @@ -83,10 +84,12 @@ const ( ) type modelCooldownError struct { - model string - resetIn time.Duration - provider string - cause error + model string + resetIn time.Duration + provider string + cause error + claudeCooldown bool + claudeSignals map[string]string } // NewModelCooldownError creates an error representing model-level cooldown. @@ -99,17 +102,27 @@ func newModelCooldownError(model, provider string, resetIn time.Duration) *model } func newModelCooldownErrorWithCause(model, provider string, resetIn time.Duration, cause error) *modelCooldownError { + return newModelCooldownErrorForCandidates(model, provider, resetIn, cause, isClaudeCooldownProvider(provider), nil) +} + +func newModelCooldownErrorForCandidates(model, provider string, resetIn time.Duration, cause error, claudeCooldown bool, claudeSignals map[string]string) *modelCooldownError { if resetIn < 0 { resetIn = 0 } return &modelCooldownError{ - model: model, - provider: provider, - resetIn: resetIn, - cause: cause, + model: model, + provider: provider, + resetIn: resetIn, + cause: cause, + claudeCooldown: claudeCooldown, + claudeSignals: maps.Clone(claudeSignals), } } +func isClaudeCooldownProvider(provider string) bool { + return strings.EqualFold(strings.TrimSpace(provider), "claude") +} + func (e *modelCooldownError) IsModelCooldown() bool { return true } @@ -352,17 +365,419 @@ func (e *modelCooldownError) StatusCode() int { return http.StatusTooManyRequests } -func (e *modelCooldownError) Headers() http.Header { +func (e *modelCooldownError) resetSeconds() int64 { + if e == nil || e.resetIn <= 0 { + return 0 + } + seconds := int64(e.resetIn / time.Second) + if e.resetIn%time.Second > 0 { + seconds++ + } + return seconds +} + +// SafeResponseHeaders exposes only the locally trusted cooldown headers. +func (e *modelCooldownError) SafeResponseHeaders() http.Header { headers := make(http.Header) headers.Set("Content-Type", "application/json") - resetSeconds := int(math.Ceil(e.resetIn.Seconds())) - if resetSeconds < 0 { - resetSeconds = 0 - } - headers.Set("Retry-After", strconv.Itoa(resetSeconds)) + headers.Set("Retry-After", strconv.FormatInt(e.resetSeconds(), 10)) return headers } +func (e *modelCooldownError) Headers() http.Header { + headers := e.SafeResponseHeaders() + if e != nil && e.claudeCooldown && e.resetIn > 0 { + e.appendClaudeUnifiedHeaders(headers) + } + return headers +} + +func (e *modelCooldownError) appendClaudeUnifiedHeaders(headers http.Header) { + resetSeconds := e.resetSeconds() + if resetSeconds <= 0 { + return + } + resetAt := time.Now().Unix() + resetSeconds + resetStr := strconv.FormatInt(resetAt, 10) + + if hasRejectedClaudeUnifiedSignals(e.claudeSignals) { + for key, value := range e.claudeSignals { + if strings.HasPrefix(strings.ToLower(strings.TrimSpace(key)), "anthropic-ratelimit-unified-") { + headers.Set(key, value) + } + } + headers.Set("Anthropic-Ratelimit-Unified-Status", "rejected") + headers.Set("Anthropic-Ratelimit-Unified-Reset", resetStr) + + rejWindows := rejectedClaudeUnifiedWindows(headers) + claim := strings.ToLower(strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-Representative-Claim"))) + if len(rejWindows) == 0 && claim == "" && isClaudeOverageOnlyHeaders(headers) { + claim = "overage" + headers.Set("Anthropic-Ratelimit-Unified-Representative-Claim", claim) + } + if len(rejWindows) == 0 && claim != "overage" { + fallbackWindow := fallbackClaudeWindowForClaim(headers, claim, e.resetIn) + if fallbackWindow != "" { + statusHeader := claudeUnifiedWindowHeader(fallbackWindow, "Status") + if strings.TrimSpace(headers.Get(statusHeader)) == "" { + headers.Set(statusHeader, "rejected") + rejWindows = append(rejWindows, fallbackWindow) + } + } + } + + if len(rejWindows) == 1 { + headers.Set(claudeUnifiedWindowHeader(rejWindows[0], "Reset"), resetStr) + } else if len(rejWindows) > 1 { + latestHeader := "" + var latestEpoch int64 + for _, window := range rejWindows { + resetHeader := claudeUnifiedWindowHeader(window, "Reset") + raw := strings.TrimSpace(headers.Get(resetHeader)) + if raw == "" { + headers.Set(resetHeader, resetStr) + continue + } + if parsed, errParse := strconv.ParseInt(raw, 10, 64); errParse == nil && (latestHeader == "" || parsed >= latestEpoch) { + latestHeader = resetHeader + latestEpoch = parsed + } + } + if latestHeader != "" { + headers.Set(latestHeader, resetStr) + } + } + + if claim == "" { + if inferred := inferClaudeRepresentativeClaim(headers, rejWindows, e.resetIn); inferred != "" { + headers.Set("Anthropic-Ratelimit-Unified-Representative-Claim", inferred) + } + } + if strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-Overage-Status")) == "" { + headers.Set("Anthropic-Ratelimit-Unified-Overage-Status", "rejected") + } + return + } + + claim := "five_hour" + if e.resetIn > 5*time.Hour { + claim = "seven_day" + } + headers.Set("Anthropic-Ratelimit-Unified-Status", "rejected") + headers.Set("Anthropic-Ratelimit-Unified-Reset", resetStr) + headers.Set("Anthropic-Ratelimit-Unified-Representative-Claim", claim) + headers.Set("Anthropic-Ratelimit-Unified-Overage-Status", "rejected") + if claim == "five_hour" { + headers.Set("Anthropic-Ratelimit-Unified-5h-Status", "rejected") + headers.Set("Anthropic-Ratelimit-Unified-5h-Reset", resetStr) + } else { + headers.Set("Anthropic-Ratelimit-Unified-7d-Status", "rejected") + headers.Set("Anthropic-Ratelimit-Unified-7d-Reset", resetStr) + } +} + +func claudeUnifiedWindowName(headerKey string) (string, bool) { + lower := strings.ToLower(strings.TrimSpace(headerKey)) + rest, hasPrefix := strings.CutPrefix(lower, "anthropic-ratelimit-unified-") + if !hasPrefix { + return "", false + } + window, hasSuffix := strings.CutSuffix(rest, "-status") + if !hasSuffix || window == "" || window == "overage" { + return "", false + } + return window, true +} + +func claudeUnifiedWindowHeader(window, field string) string { + return http.CanonicalHeaderKey("Anthropic-Ratelimit-Unified-" + window + "-" + field) +} + +func rejectedClaudeUnifiedWindows(headers http.Header) []string { + var windows []string + for key := range headers { + window, ok := claudeUnifiedWindowName(key) + if ok && strings.EqualFold(strings.TrimSpace(headers.Get(key)), "rejected") { + windows = append(windows, window) + } + } + sort.Strings(windows) + return windows +} + +func isAllowedClaudeWindowStatus(status string) bool { + lower := strings.ToLower(strings.TrimSpace(status)) + return lower == "allowed" || lower == "allowed_warning" +} + +func isClaudeUtilizationBelowCap(raw string) bool { + raw = strings.TrimSpace(raw) + if raw == "" { + return false + } + u, errParse := strconv.ParseFloat(raw, 64) + if errParse != nil || math.IsNaN(u) || math.IsInf(u, 0) { + return false + } + return u >= 0 && u < 1.0 +} + +func isClaudeOverageOnlyHeaders(headers http.Header) bool { + overageRejected := strings.EqualFold(strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-Overage-Status")), "rejected") || + strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-Overage-Disabled-Reason")) != "" + if !overageRejected { + return false + } + status5h := strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-5h-Status")) + status7d := strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-7d-Status")) + allowed5h := isAllowedClaudeWindowStatus(status5h) + allowed7d := isAllowedClaudeWindowStatus(status7d) + if allowed5h && allowed7d { + return true + } + if allowed7d && status5h == "" { + util5h := strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-5h-Utilization")) + if util5h == "" || isClaudeUtilizationBelowCap(util5h) { + return true + } + } + if allowed5h && status7d == "" && isClaudeUtilizationBelowCap(headers.Get("Anthropic-Ratelimit-Unified-7d-Utilization")) { + return true + } + return false +} + +func fallbackClaudeWindowForClaim(headers http.Header, claim string, resetIn time.Duration) string { + switch { + case claim == "seven_day_overage_included": + return "7d_oi" + case strings.HasPrefix(claim, "seven_day_"): + if suffix := strings.TrimPrefix(claim, "seven_day_"); suffix != "" { + return "7d_" + suffix + } + case claim == "seven_day": + return "7d" + case claim == "five_hour": + return "5h" + case claim == "": + allowed5h := isAllowedClaudeWindowStatus(headers.Get("Anthropic-Ratelimit-Unified-5h-Status")) + allowed7d := isAllowedClaudeWindowStatus(headers.Get("Anthropic-Ratelimit-Unified-7d-Status")) + switch { + case allowed5h && allowed7d: + return "" + case allowed5h: + return "7d" + case allowed7d: + return "5h" + case resetIn > 5*time.Hour: + return "7d" + default: + return "5h" + } + } + return "" +} + +func inferClaudeRepresentativeClaim(headers http.Header, windows []string, resetIn time.Duration) string { + has5h := false + has7d := false + has7dOI := false + modelFamily7d := "" + otherWindow := "" + for _, window := range windows { + switch { + case window == "7d_oi": + has7dOI = true + case strings.HasPrefix(window, "7d_"): + if modelFamily7d == "" { + modelFamily7d = "seven_day_" + strings.TrimPrefix(window, "7d_") + } + case window == "7d": + has7d = true + case window == "5h": + has5h = true + default: + if otherWindow == "" { + otherWindow = window + } + } + } + switch { + case has7dOI: + return "seven_day_overage_included" + case modelFamily7d != "": + return modelFamily7d + case has7d: + return "seven_day" + case has5h: + return "five_hour" + case otherWindow != "": + return otherWindow + } + if strings.EqualFold(strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-Overage-Status")), "rejected") || + strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-Overage-Disabled-Reason")) != "" { + return "overage" + } + allowed5h := isAllowedClaudeWindowStatus(headers.Get("Anthropic-Ratelimit-Unified-5h-Status")) + allowed7d := isAllowedClaudeWindowStatus(headers.Get("Anthropic-Ratelimit-Unified-7d-Status")) + switch { + case allowed5h && allowed7d: + return "" + case allowed5h: + return "seven_day" + case allowed7d: + return "five_hour" + case resetIn > 5*time.Hour: + return "seven_day" + default: + return "five_hour" + } +} + +func hasRejectedClaudeUnifiedSignals(signals map[string]string) bool { + if len(signals) == 0 { + return false + } + if strings.EqualFold(strings.TrimSpace(signals["Anthropic-Ratelimit-Unified-Status"]), "rejected") { + return true + } + for key, val := range signals { + if _, ok := claudeUnifiedWindowName(key); ok && strings.EqualFold(strings.TrimSpace(val), "rejected") { + return true + } + } + return false +} + +func claudeSignalsFromEvaluatedQuota(q QuotaState) (bool, map[string]string) { + if hasRejectedClaudeUnifiedSignals(q.cooldownSignals) { + return true, maps.Clone(q.cooldownSignals) + } + if q.cooldownEvaluated { + return false, nil + } + if hasRejectedClaudeUnifiedSignals(q.Signals) { + return true, maps.Clone(q.Signals) + } + return false, nil +} + +func claudeUnifiedCooldownForAuth(auth *Auth, model string, now time.Time) (bool, map[string]string) { + if auth == nil || executorKeyFromAuth(auth) != "claude" { + return false, nil + } + if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now) { + state := existingModelState(auth, model) + if state != nil && state.NextRetryAfter.After(auth.Quota.NextRecoverAt) { + if shadow := state.credentialQuotaShadow; shadow != nil { + if !shadow.Quota.Exceeded || !shadow.Quota.NextRecoverAt.After(now) || shadow.NextRetryAfter.After(shadow.Quota.NextRecoverAt) { + return false, nil + } + if strings.EqualFold(strings.TrimSpace(shadow.Quota.Reason), "cloudflare challenge") { + return false, nil + } + return claudeSignalsFromEvaluatedQuota(shadow.Quota) + } + return false, nil + } + if ok, signals := claudeSignalsFromEvaluatedQuota(auth.Quota); ok { + return true, signals + } + if state != nil && !state.Quota.NextRecoverAt.Before(auth.Quota.NextRecoverAt) { + if ok, signals := claudeSignalsFromEvaluatedQuota(state.Quota); ok { + return true, signals + } + } + return true, nil + } + if len(auth.ModelStates) > 0 { + if model == "" { + if !auth.Unavailable { + return false, nil + } + keys := make([]string, 0, len(auth.ModelStates)) + for key := range auth.ModelStates { + keys = append(keys, key) + } + sort.Strings(keys) + var earliestRecover time.Time + var earliestSignals map[string]string + found := false + for _, key := range keys { + st := auth.ModelStates[key] + if st != nil && (st.credentialQuotaPropagated || st.Quota.Reason == "credential_quota") { + st = st.credentialQuotaShadow + } + if st == nil || !st.Quota.Exceeded || !st.Quota.NextRecoverAt.After(now) || st.NextRetryAfter.After(st.Quota.NextRecoverAt) { + return false, nil + } + if strings.EqualFold(strings.TrimSpace(st.Quota.Reason), "cloudflare challenge") { + return false, nil + } + okSt, sig := claudeSignalsFromEvaluatedQuota(st.Quota) + if !okSt { + return false, nil + } + if !found || st.Quota.NextRecoverAt.Before(earliestRecover) { + found = true + earliestRecover = st.Quota.NextRecoverAt + earliestSignals = sig + } + } + return found, earliestSignals + } + state := existingModelState(auth, model) + if state != nil && (state.credentialQuotaPropagated || state.Quota.Reason == "credential_quota") { + state = state.credentialQuotaShadow + } + if state == nil || !state.Quota.Exceeded || !state.Quota.NextRecoverAt.After(now) || state.NextRetryAfter.After(state.Quota.NextRecoverAt) { + return false, nil + } + if strings.EqualFold(strings.TrimSpace(state.Quota.Reason), "cloudflare challenge") { + return false, nil + } + return claudeSignalsFromEvaluatedQuota(state.Quota) + } + if !auth.Quota.Exceeded || !auth.Quota.NextRecoverAt.After(now) || auth.NextRetryAfter.After(auth.Quota.NextRecoverAt) { + return false, nil + } + if strings.EqualFold(strings.TrimSpace(auth.Quota.Reason), "cloudflare challenge") { + return false, nil + } + return claudeSignalsFromEvaluatedQuota(auth.Quota) +} + +func claudeCooldownMetadataForAuths(ctx context.Context, auths []*Auth, modelForAuth func(*Auth) string, now time.Time, clientHeaders ...http.Header) (bool, map[string]string) { + if len(auths) == 0 { + return false, nil + } + var earliestAuth *Auth + var earliestSignals map[string]string + var earliestNext time.Time + for _, candidate := range auths { + checkModel := "" + if modelForAuth != nil { + checkModel = modelForAuth(candidate) + } + okClaude, signals := claudeUnifiedCooldownForAuth(candidate, checkModel, now) + if !okClaude { + return false, nil + } + blocked, reason, next := isAuthBlockedForModelForRequest(ctx, candidate, checkModel, now, clientHeaders...) + if blocked && reason == blockReasonCooldown && next.After(now) { + if earliestAuth == nil || next.Before(earliestNext) || (next.Equal(earliestNext) && candidate.ID < earliestAuth.ID) { + earliestAuth = candidate + earliestSignals = signals + earliestNext = next + } + } + } + if earliestAuth == nil { + return false, nil + } + return true, earliestSignals +} + func authPriority(auth *Auth) int { if auth == nil || auth.Attributes == nil { return 0 @@ -558,7 +973,10 @@ func getAvailableAuthsWithPriorityModeForRequest(ctx context.Context, auths []*A if resetIn < 0 { resetIn = 0 } - return nil, newModelCooldownError(model, providerForError, resetIn) + claudeCooldown, claudeSignals := claudeCooldownMetadataForAuths(ctx, auths, func(*Auth) string { + return model + }, now, clientHeaders...) + return nil, newModelCooldownErrorForCandidates(model, providerForError, resetIn, nil, claudeCooldown, claudeSignals) } return nil, newAuthUnavailableError(earliest, now) } @@ -819,7 +1237,23 @@ func isAuthBlockedForModelForRequest(ctx context.Context, auth *Auth, model stri return true, blockReasonOther, time.Time{} } if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now) { - return true, blockReasonCooldown, auth.Quota.NextRecoverAt + credNext := auth.Quota.NextRecoverAt + if model != "" && len(auth.ModelStates) > 0 { + modelKey := canonicalModelLookupKey(model) + for stateModel, state := range auth.ModelStates { + if state == nil || canonicalModelLookupKey(stateModel) != modelKey { + continue + } + if state.Status == StatusDisabled { + return true, blockReasonDisabled, time.Time{} + } + stateBlocked, _, next := availabilityBlock(state.Unavailable, state.Quota.Exceeded, state.NextRetryAfter, state.Quota.NextRecoverAt, now) + if stateBlocked && next.After(credNext) { + credNext = next + } + } + } + return true, blockReasonCooldown, credNext } unavailable := codexRefreshUnavailableForRequest(ctx, auth, now, clientHeaders...) if model != "" { diff --git a/sdk/cliproxy/auth/types.go b/sdk/cliproxy/auth/types.go index 7cf8d91aa7..ea18ce0106 100644 --- a/sdk/cliproxy/auth/types.go +++ b/sdk/cliproxy/auth/types.go @@ -355,6 +355,10 @@ type QuotaState struct { // Cooldown transitions must use applyCooldownFields so they cannot replace // this snapshot. Signals map[string]string `json:"signals,omitempty"` + // cooldownSignals retains rejected quota headers captured when the active + // cooldown was recorded so out-of-order completions cannot overwrite them. + cooldownSignals map[string]string + cooldownEvaluated bool } // Clone returns an independent copy of the quota state. @@ -366,6 +370,12 @@ func (q QuotaState) Clone() QuotaState { copyQuota.Signals[key] = value } } + if len(q.cooldownSignals) > 0 { + copyQuota.cooldownSignals = make(map[string]string, len(q.cooldownSignals)) + for key, value := range q.cooldownSignals { + copyQuota.cooldownSignals[key] = value + } + } return copyQuota }