fix(claude): emit anthropic-ratelimit-unified headers on local cooldown 429s

- Retain and persist rejected Claude unified quota signals alongside active cooldown deadlines so out-of-order completions, restarts, and dual-layer quota wraps keep accurate window metadata.
- Populate anthropic-ratelimit-unified-* headers on Claude quota modelCooldownError responses aligned with Retry-After while keeping SafeResponseHeaders restricted to trusted retry headers.
- Exclude non-Claude, Cloudflare challenge, and ordinary per-minute rate-limit cooldowns from unified subscription quota header synthesis across selector and scheduler paths.

Closes: #7417
Co-authored-by: Dylan <1990016@gmail.com>
Co-authored-by: lzt404 <2596933790@qq.com>
This commit is contained in:
Luis Pater
2026-11-11 01:19:00 +08:00
parent effbac168a
commit cb70eacfa5
10 changed files with 2043 additions and 75 deletions

View File

@@ -224,3 +224,101 @@ func TestClaudeCooldownMessagesRoute(t *testing.T) {
})
}
}
type cooldownClaudeTestExecutor struct {
preFTKeepAliveClaudeExecutor
}
func (*cooldownClaudeTestExecutor) Identifier() string { return "claude" }
func TestClaudeCooldownMessagesRoute_PassthroughUnifiedRateLimitHeaders(t *testing.T) {
gin.SetMode(gin.TestMode)
chunks := make(chan coreexecutor.StreamChunk)
close(chunks)
executor := &cooldownClaudeTestExecutor{preFTKeepAliveClaudeExecutor: preFTKeepAliveClaudeExecutor{chunks: chunks}}
manager := coreauth.NewManager(nil, coreauth.NewSessionAffinitySelector(&coreauth.RoundRobinSelector{}), nil)
manager.RegisterExecutor(executor)
const model = "issue-7417-cooldown-route-model"
const authID = "issue-7417-cooldown-route-auth"
next := time.Now().Add(7823 * time.Second)
auth := &coreauth.Auth{
ID: authID,
Provider: executor.Identifier(),
Status: coreauth.StatusActive,
Unavailable: true,
NextRetryAfter: next,
Quota: coreauth.QuotaState{
Exceeded: true,
Reason: "credential_quota",
NextRecoverAt: next,
Signals: map[string]string{
"Anthropic-Ratelimit-Unified-Status": "rejected",
"Anthropic-Ratelimit-Unified-5h-Status": "rejected",
"Anthropic-Ratelimit-Unified-Reset": strconv.FormatInt(next.Add(-10*time.Second).Unix(), 10),
},
},
LastError: &coreauth.Error{Code: "rate_limit_error", Message: "upstream quota exceeded", HTTPStatus: http.StatusTooManyRequests},
}
if _, errRegister := manager.Register(context.Background(), auth); errRegister != nil {
t.Fatalf("register auth: %v", errRegister)
}
registry.GetGlobalRegistry().RegisterClient(authID, executor.Identifier(), []*registry.ModelInfo{{ID: model}})
t.Cleanup(func() {
registry.GetGlobalRegistry().UnregisterClient(authID)
})
for _, passthrough := range []bool{false, true} {
for _, streaming := range []bool{false, true} {
name := "passthrough_" + strconv.FormatBool(passthrough) + "_stream_" + strconv.FormatBool(streaming)
t.Run(name, func(t *testing.T) {
handler := NewClaudeCodeAPIHandler(handlers.NewBaseAPIHandlers(&sdkconfig.SDKConfig{PassthroughHeaders: passthrough}, manager))
router := gin.New()
router.POST("/v1/messages", handler.ClaudeMessages)
requestBody := `{"model":"` + model + `","messages":[{"role":"user","content":"hi"}],"stream":` + strconv.FormatBool(streaming) + `}`
request := httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(requestBody))
request.Header.Set("Content-Type", "application/json")
recorder := httptest.NewRecorder()
before := time.Now().Unix()
router.ServeHTTP(recorder, request)
after := time.Now().Unix()
if recorder.Code != http.StatusTooManyRequests {
t.Fatalf("status = %d, want 429; body=%s", recorder.Code, recorder.Body.String())
}
seconds, errSeconds := strconv.ParseInt(recorder.Header().Get("Retry-After"), 10, 64)
if errSeconds != nil || seconds <= 0 {
t.Fatalf("Retry-After = %q, want positive seconds", recorder.Header().Get("Retry-After"))
}
if !passthrough {
if got := recorder.Header().Get("Anthropic-Ratelimit-Unified-Status"); got != "" {
t.Fatalf("passthrough=false leaked Anthropic-Ratelimit-Unified-Status = %q", got)
}
return
}
if got := recorder.Header().Get("Anthropic-Ratelimit-Unified-Status"); got != "rejected" {
t.Fatalf("Anthropic-Ratelimit-Unified-Status = %q, want rejected", got)
}
if got := recorder.Header().Get("Anthropic-Ratelimit-Unified-Representative-Claim"); got != "five_hour" {
t.Fatalf("Anthropic-Ratelimit-Unified-Representative-Claim = %q, want five_hour", got)
}
if got := recorder.Header().Get("Anthropic-Ratelimit-Unified-Overage-Status"); got != "rejected" {
t.Fatalf("Anthropic-Ratelimit-Unified-Overage-Status = %q, want rejected", got)
}
if got := recorder.Header().Get("Anthropic-Ratelimit-Unified-5h-Status"); got != "rejected" {
t.Fatalf("Anthropic-Ratelimit-Unified-5h-Status = %q, want rejected", got)
}
resetAt, errReset := strconv.ParseInt(recorder.Header().Get("Anthropic-Ratelimit-Unified-Reset"), 10, 64)
if errReset != nil || resetAt < before+seconds || resetAt > after+seconds {
t.Fatalf("Anthropic-Ratelimit-Unified-Reset = %q, want unix epoch in [%d, %d]", recorder.Header().Get("Anthropic-Ratelimit-Unified-Reset"), before+seconds, after+seconds)
}
if got := recorder.Header().Get("Anthropic-Ratelimit-Unified-5h-Reset"); got != strconv.FormatInt(resetAt, 10) {
t.Fatalf("Anthropic-Ratelimit-Unified-5h-Reset = %q, want %d", got, resetAt)
}
})
}
}
}

View File

@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"io"
"maps"
"net"
"net/http"
"regexp"
@@ -382,11 +383,18 @@ func (m *Manager) restoreCooldownRecordLocked(record CooldownStateRecord, now ti
if quota.Exceeded && quota.NextRecoverAt.IsZero() {
quota.NextRecoverAt = record.NextRetryAfter
}
quota.cooldownEvaluated = quota.Exceeded
if executorKeyFromAuth(auth) == "claude" || strings.EqualFold(strings.TrimSpace(record.Provider), "claude") {
quota.cooldownSignals = sanitizePersistedClaudeCooldownSignals(record.CooldownSignals)
}
if model == "" {
auth.Unavailable = true
auth.Status = StatusError
auth.NextRetryAfter = record.NextRetryAfter
if len(quota.cooldownSignals) == 0 && auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(now) && !quota.NextRecoverAt.After(auth.Quota.NextRecoverAt) {
quota.cooldownSignals = auth.Quota.cooldownSignals
}
applyCooldownFields(&auth.Quota, quota)
auth.Quota = mergeQuotaObservation(auth.Quota, quota)
auth.Generation++
@@ -453,10 +461,11 @@ func (m *Manager) restoreCooldownRecordLocked(record CooldownStateRecord, now ti
}
state.credentialQuotaPropagated = true
applyCooldownFields(&state.Quota, QuotaState{
Exceeded: true,
Reason: "credential_quota",
NextRecoverAt: auth.Quota.NextRecoverAt,
BackoffLevel: auth.Quota.BackoffLevel,
Exceeded: true,
Reason: "credential_quota",
NextRecoverAt: auth.Quota.NextRecoverAt,
BackoffLevel: auth.Quota.BackoffLevel,
cooldownSignals: auth.Quota.cooldownSignals,
})
if state.NextRetryAfter.Before(auth.Quota.NextRecoverAt) {
state.NextRetryAfter = auth.Quota.NextRecoverAt
@@ -466,7 +475,12 @@ func (m *Manager) restoreCooldownRecordLocked(record CooldownStateRecord, now ti
} else if isWrappedRecord {
state.credentialQuotaPropagated = true
if record.IndependentState != nil && (liveShadow == nil || !liveShadow.UpdatedAt.After(record.IndependentState.UpdatedAt)) {
state.credentialQuotaShadow = record.IndependentState.cloneShallow()
restoredShadow := record.IndependentState.cloneShallow()
restoredShadow.Quota.cooldownEvaluated = restoredShadow.Quota.Exceeded
if executorKeyFromAuth(auth) == "claude" || strings.EqualFold(strings.TrimSpace(record.Provider), "claude") {
restoredShadow.Quota.cooldownSignals = sanitizePersistedClaudeCooldownSignals(record.IndependentCooldownSignals)
}
state.credentialQuotaShadow = restoredShadow
} else if liveShadow != nil {
state.credentialQuotaShadow = liveShadow
}
@@ -756,6 +770,9 @@ func cooldownStateRecordEqual(a, b CooldownStateRecord) bool {
if !cooldownErrorEqual(a.LastError, b.LastError) {
return false
}
if !maps.Equal(a.CooldownSignals, b.CooldownSignals) || !maps.Equal(a.IndependentCooldownSignals, b.IndependentCooldownSignals) {
return false
}
return cooldownIndependentStateEqual(a.IndependentState, b.IndependentState)
}
@@ -793,16 +810,21 @@ func authCooldownStateRecord(auth *Auth, now time.Time) (CooldownStateRecord, bo
if auth == nil || !auth.Unavailable || auth.NextRetryAfter.IsZero() || !auth.NextRetryAfter.After(now) {
return CooldownStateRecord{}, false
}
var cooldownSignals map[string]string
if auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(now) && !auth.NextRetryAfter.After(auth.Quota.NextRecoverAt) {
cooldownSignals = sanitizePersistedClaudeCooldownSignals(auth.Quota.cooldownSignals)
}
return CooldownStateRecord{
Provider: strings.TrimSpace(auth.Provider),
AuthID: auth.ID,
AuthFile: cooldownAuthFile(auth),
Status: "cooling",
NextRetryAfter: auth.NextRetryAfter,
Reason: cooldownReason(auth.StatusMessage, auth.Quota, auth.LastError),
Quota: cooldownFieldsOf(auth.Quota),
LastError: cloneError(auth.LastError),
UpdatedAt: auth.UpdatedAt,
Provider: strings.TrimSpace(auth.Provider),
AuthID: auth.ID,
AuthFile: cooldownAuthFile(auth),
Status: "cooling",
NextRetryAfter: auth.NextRetryAfter,
Reason: cooldownReason(auth.StatusMessage, auth.Quota, auth.LastError),
Quota: cooldownFieldsOf(auth.Quota),
LastError: cloneError(auth.LastError),
UpdatedAt: auth.UpdatedAt,
CooldownSignals: cooldownSignals,
}, true
}
@@ -811,20 +833,32 @@ func modelCooldownStateRecord(auth *Auth, model string, state *ModelState, now t
if auth == nil || state == nil || model == "" || !state.Unavailable || state.NextRetryAfter.IsZero() || !state.NextRetryAfter.After(now) {
return CooldownStateRecord{}, false
}
var cooldownSignals map[string]string
liveCredentialWrap := auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now)
expiredCredentialWrap := (state.credentialQuotaPropagated || state.Quota.Reason == "credential_quota") && !liveCredentialWrap
if !expiredCredentialWrap && state.Quota.Exceeded && state.Quota.NextRecoverAt.After(now) &&
(state.Quota.Reason == "credential_quota" || !state.NextRetryAfter.After(state.Quota.NextRecoverAt)) {
cooldownSignals = sanitizePersistedClaudeCooldownSignals(state.Quota.cooldownSignals)
}
record := CooldownStateRecord{
Provider: strings.TrimSpace(auth.Provider),
AuthID: auth.ID,
AuthFile: cooldownAuthFile(auth),
Model: model,
Status: "cooling",
NextRetryAfter: state.NextRetryAfter,
Reason: cooldownReason(state.StatusMessage, state.Quota, state.LastError),
Quota: cooldownFieldsOf(state.Quota),
LastError: cloneError(state.LastError),
UpdatedAt: state.UpdatedAt,
Provider: strings.TrimSpace(auth.Provider),
AuthID: auth.ID,
AuthFile: cooldownAuthFile(auth),
Model: model,
Status: "cooling",
NextRetryAfter: state.NextRetryAfter,
Reason: cooldownReason(state.StatusMessage, state.Quota, state.LastError),
Quota: cooldownFieldsOf(state.Quota),
LastError: cloneError(state.LastError),
UpdatedAt: state.UpdatedAt,
CooldownSignals: cooldownSignals,
}
if state.credentialQuotaShadow != nil {
record.IndependentState = state.credentialQuotaShadow.cloneShallow()
if state.credentialQuotaShadow.Quota.Exceeded && state.credentialQuotaShadow.Quota.NextRecoverAt.After(now) &&
!state.credentialQuotaShadow.NextRetryAfter.After(state.credentialQuotaShadow.Quota.NextRecoverAt) {
record.IndependentCooldownSignals = sanitizePersistedClaudeCooldownSignals(state.credentialQuotaShadow.Quota.cooldownSignals)
}
}
record.CredentialQuotaPropagated = state.credentialQuotaPropagated
return record, true
@@ -1123,8 +1157,12 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
state.NextRetryAfter = next
}
case 429:
prevStateQuotaActive := state.Quota.Exceeded && state.Quota.NextRecoverAt.After(now)
prevStateRecoverAt := state.Quota.NextRecoverAt
prevStateCooldownSignals := maps.Clone(state.Quota.cooldownSignals)
var next time.Time
var credentialNext time.Time
keptPriorStateQuota := false
backoffLevel := state.Quota.BackoffLevel
if result.CredentialScope {
backoffLevel = 0
@@ -1152,16 +1190,32 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
next, backoffLevel = quotaCooldownAfterFailure(quotaForFailure, now)
}
credentialNext = next
if state.Quota.Exceeded && state.Quota.NextRecoverAt.After(next) {
next = state.Quota.NextRecoverAt
if prevStateQuotaActive && (prevStateRecoverAt.After(next) || (!result.CredentialScope && result.RetryAfter == nil)) {
next = prevStateRecoverAt
keptPriorStateQuota = true
} else if result.CredentialScope && result.RetryAfter == nil && auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now) {
// quotaCooldownAfterFailure reused the open credential_quota window.
keptPriorStateQuota = prevStateQuotaActive && prevStateRecoverAt.Equal(next)
}
}
var cooldownSignals map[string]string
if !disableCooling {
if keptPriorStateQuota {
cooldownSignals = prevStateCooldownSignals
} else {
cooldownSignals = rejectedClaudeSignalsForCooldown(result.Provider, responseHeaders)
if len(cooldownSignals) == 0 && prevStateQuotaActive && prevStateRecoverAt.Equal(next) {
cooldownSignals = prevStateCooldownSignals
}
}
}
state.NextRetryAfter = next
applyCooldownFields(&state.Quota, QuotaState{
Exceeded: true,
Reason: "quota",
NextRecoverAt: next,
BackoffLevel: backoffLevel,
Exceeded: true,
Reason: "quota",
NextRecoverAt: next,
BackoffLevel: backoffLevel,
cooldownSignals: cooldownSignals,
})
// Non-credential own failures re-derive their independent layer
// below, so a stale shadow is dropped. A credential-scoped
@@ -1187,12 +1241,24 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
}
}
if result.CredentialScope && !disableCooling {
prevAuthCredActive := auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now)
prevAuthCredRecoverAt := auth.Quota.NextRecoverAt
prevAuthCredSignals := maps.Clone(auth.Quota.cooldownSignals)
var credCooldownSignals map[string]string
if prevAuthCredActive && (prevAuthCredRecoverAt.After(credentialNext) || result.RetryAfter == nil) {
credCooldownSignals = prevAuthCredSignals
} else {
credCooldownSignals = rejectedClaudeSignalsForCooldown(result.Provider, responseHeaders)
if len(credCooldownSignals) == 0 && prevAuthCredActive && prevAuthCredRecoverAt.Equal(credentialNext) {
credCooldownSignals = prevAuthCredSignals
}
}
for _, otherState := range auth.ModelStates {
if otherState != nil && otherState != state {
// Normalize a naturally expired wrap first (issue #6690) so a
// stale marker cannot suppress snapshotting a fresh block.
if (otherState.credentialQuotaPropagated || otherState.Quota.Reason == "credential_quota") &&
!(auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now)) {
!prevAuthCredActive {
if shadow := otherState.credentialQuotaShadow; shadow != nil && isModelStateActiveCooldown(shadow, now) {
preservedSeq := shadow.cooldownRecordedSeq
*otherState = *shadow.cloneShallow()
@@ -1219,8 +1285,13 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
otherState.Unavailable = true
otherState.Status = StatusError
otherQuotaNext := credentialNext
otherSignals := credCooldownSignals
if prevAuthCredActive && prevAuthCredRecoverAt.After(otherQuotaNext) {
otherQuotaNext = prevAuthCredRecoverAt
}
if otherState.Quota.Exceeded && otherState.Quota.NextRecoverAt.After(otherQuotaNext) {
otherQuotaNext = otherState.Quota.NextRecoverAt
otherSignals = maps.Clone(otherState.Quota.cooldownSignals)
}
otherRetryAfter := otherQuotaNext
// Propagation only extends a sibling's still-live
@@ -1230,25 +1301,34 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
}
otherState.NextRetryAfter = otherRetryAfter
applyCooldownFields(&otherState.Quota, QuotaState{
Exceeded: true,
Reason: "credential_quota",
NextRecoverAt: otherQuotaNext,
BackoffLevel: backoffLevel,
Exceeded: true,
Reason: "credential_quota",
NextRecoverAt: otherQuotaNext,
BackoffLevel: backoffLevel,
cooldownSignals: otherSignals,
})
}
}
if !wasTerminalUnauthorized {
auth.Unavailable = true
authNext := credentialNext
if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" &&
auth.Quota.NextRecoverAt.After(authNext) {
authNext = auth.Quota.NextRecoverAt
if prevAuthCredActive && prevAuthCredRecoverAt.After(authNext) {
authNext = prevAuthCredRecoverAt
}
auth.Quota.Exceeded = true
auth.Quota.Reason = "credential_quota"
auth.Quota.NextRecoverAt = authNext
auth.Quota.BackoffLevel = backoffLevel
auth.Quota.cooldownSignals = maps.Clone(credCooldownSignals)
auth.Quota.cooldownEvaluated = true
auth.NextRetryAfter = authNext
if authNext.After(state.Quota.NextRecoverAt) || (prevAuthCredActive && result.RetryAfter == nil) {
state.Quota.NextRecoverAt = authNext
state.Quota.cooldownSignals = maps.Clone(credCooldownSignals)
if state.NextRetryAfter.Before(authNext) {
state.NextRetryAfter = authNext
}
}
// The reporting model is now part of the wrap too: mark it
// so a later credential report from another model cannot
// snapshot this credential-derived state as independent.
@@ -1310,10 +1390,11 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
}
if !(outer.Quota.Exceeded && outer.Quota.Reason == "credential_quota") {
applyCooldownFields(&outer.Quota, QuotaState{
Exceeded: true,
Reason: "credential_quota",
NextRecoverAt: auth.Quota.NextRecoverAt,
BackoffLevel: auth.Quota.BackoffLevel,
Exceeded: true,
Reason: "credential_quota",
NextRecoverAt: auth.Quota.NextRecoverAt,
BackoffLevel: auth.Quota.BackoffLevel,
cooldownSignals: auth.Quota.cooldownSignals,
})
}
*outerState = *outer
@@ -1337,7 +1418,36 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
disableCooling = false
}
if !wasTerminalUnauthorized {
prevAuthQuotaActive := auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(now)
prevAuthRecoverAt := auth.Quota.NextRecoverAt
prevAuthCooldownSignals := maps.Clone(auth.Quota.cooldownSignals)
keptPriorAuthQuota := false
if !disableCooling && statusCodeFromResult(result.Error) == 429 && prevAuthQuotaActive {
if result.RetryAfter == nil {
keptPriorAuthQuota = true
} else {
cooldown := *result.RetryAfter
if cooldown < minQuotaCooldownFloor {
cooldown = minQuotaCooldownFloor
}
if prevAuthRecoverAt.After(now.Add(cooldown).Round(0)) {
keptPriorAuthQuota = true
}
}
}
applyAuthFailureState(auth, result.Error, result.RetryAfter, now, disableCooling)
if !disableCooling && statusCodeFromResult(result.Error) == 429 && auth.Quota.Exceeded {
auth.Quota.cooldownEvaluated = true
if keptPriorAuthQuota {
auth.Quota.cooldownSignals = prevAuthCooldownSignals
} else if rejectedSignals := rejectedClaudeSignalsForCooldown(result.Provider, responseHeaders); len(rejectedSignals) > 0 {
auth.Quota.cooldownSignals = rejectedSignals
} else if prevAuthQuotaActive && auth.Quota.NextRecoverAt.Equal(prevAuthRecoverAt) {
auth.Quota.cooldownSignals = prevAuthCooldownSignals
} else {
auth.Quota.cooldownSignals = nil
}
}
auth.cooldownRecordedSeq = m.currentExecutionSeq()
}
}
@@ -1630,15 +1740,25 @@ func mergeModelState(target, source *ModelState) *ModelState {
NextRetryAfter: target.NextRetryAfter,
LastError: cloneError(preferred.LastError),
Quota: QuotaState{
Exceeded: target.Quota.Exceeded || source.Quota.Exceeded,
Reason: preferred.Quota.Reason,
NextRecoverAt: target.Quota.NextRecoverAt,
BackoffLevel: target.Quota.BackoffLevel,
Exceeded: target.Quota.Exceeded || source.Quota.Exceeded,
Reason: preferred.Quota.Reason,
NextRecoverAt: target.Quota.NextRecoverAt,
BackoffLevel: target.Quota.BackoffLevel,
cooldownEvaluated: target.Quota.cooldownEvaluated || source.Quota.cooldownEvaluated,
},
UpdatedAt: target.UpdatedAt,
}
merged.Quota = mergeQuotaObservation(merged.Quota, fallback.Quota)
merged.Quota = mergeQuotaObservation(merged.Quota, preferred.Quota)
if source.Quota.NextRecoverAt.After(target.Quota.NextRecoverAt) {
merged.Quota.cooldownSignals = maps.Clone(source.Quota.cooldownSignals)
} else if target.Quota.NextRecoverAt.After(source.Quota.NextRecoverAt) {
merged.Quota.cooldownSignals = maps.Clone(target.Quota.cooldownSignals)
} else if len(preferred.Quota.cooldownSignals) > 0 {
merged.Quota.cooldownSignals = maps.Clone(preferred.Quota.cooldownSignals)
} else if len(fallback.Quota.cooldownSignals) > 0 {
merged.Quota.cooldownSignals = maps.Clone(fallback.Quota.cooldownSignals)
}
if source.NextRetryAfter.After(merged.NextRetryAfter) {
merged.NextRetryAfter = source.NextRetryAfter
}
@@ -1873,10 +1993,7 @@ func updateAggregatedAvailability(auth *Auth, now time.Time) {
} else if auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(now) {
// Retain active auth-level quota cooldown
} else {
auth.Quota.Exceeded = false
auth.Quota.Reason = ""
auth.Quota.NextRecoverAt = time.Time{}
auth.Quota.BackoffLevel = 0
applyCooldownFields(&auth.Quota, QuotaState{})
}
}
@@ -1920,10 +2037,7 @@ func clearAuthStateOnSuccess(auth *Auth, now time.Time) {
auth.Unavailable = false
auth.Status = StatusActive
auth.StatusMessage = ""
auth.Quota.Exceeded = false
auth.Quota.Reason = ""
auth.Quota.NextRecoverAt = time.Time{}
auth.Quota.BackoffLevel = 0
applyCooldownFields(&auth.Quota, QuotaState{})
auth.LastError = nil
auth.NextRetryAfter = time.Time{}
auth.cooldownRecordedSeq = 0

View File

@@ -909,7 +909,10 @@ func (m *Manager) availableAuthsForRouteModelForRequest(ctx context.Context, aut
if resetIn < 0 {
resetIn = 0
}
return nil, newModelCooldownErrorWithCause(routeModel, providerForError, resetIn, lastCandidateErr)
claudeCooldown, claudeSignals := claudeCooldownMetadataForAuths(ctx, auths, func(candidate *Auth) string {
return m.selectionModelForAuth(candidate, routeModel)
}, now, clientHeaders...)
return nil, newModelCooldownErrorForCandidates(routeModel, providerForError, resetIn, lastCandidateErr, claudeCooldown, claudeSignals)
}
if unauthorizedCount == len(auths) && len(auths) > 0 {
terminalCause := latestUnauthorizedCandidateError(auths)
@@ -1024,7 +1027,14 @@ func restoreModelCooldownErrorModel(err error, requestedModel string) error {
if !errors.As(err, &cooldownErr) || cooldownErr == nil || cooldownErr.model != "" {
return err
}
return newModelCooldownErrorWithCause(requestedModel, cooldownErr.provider, cooldownErr.resetIn, cooldownErr.cause)
return newModelCooldownErrorForCandidates(
requestedModel,
cooldownErr.provider,
cooldownErr.resetIn,
cooldownErr.cause,
cooldownErr.claudeCooldown,
cooldownErr.claudeSignals,
)
}
func latestUnauthorizedCandidateError(auths []*Auth) error {

File diff suppressed because it is too large Load Diff

View File

@@ -34,6 +34,11 @@ type CooldownStateRecord struct {
// CredentialQuotaPropagated preserves wrap provenance for models that
// carry no independent layer (e.g. the reporting model itself).
CredentialQuotaPropagated bool `json:"credential_quota_propagated,omitempty"`
// CooldownSignals persists bounded rejected quota headers captured when
// this cooldown was recorded without touching passive Quota.Signals watermarks.
CooldownSignals map[string]string `json:"cooldown_signals,omitempty"`
// IndependentCooldownSignals persists CooldownSignals for IndependentState.
IndependentCooldownSignals map[string]string `json:"independent_cooldown_signals,omitempty"`
}
// CooldownStateStore persists runtime cooldown state independently from auth tokens.

View File

@@ -310,7 +310,7 @@ func SafeResponseHeaders(err error) http.Header {
}
var modelCooldown *modelCooldownError
if errors.As(err, &modelCooldown) && modelCooldown != nil {
return modelCooldown.Headers()
return modelCooldown.SafeResponseHeaders()
}
var unavailable *authUnavailableError
if errors.As(err, &unavailable) && unavailable != nil {

View File

@@ -1,6 +1,7 @@
package auth
import (
"maps"
"net/http"
"sort"
"strings"
@@ -86,6 +87,32 @@ func applyCooldownFields(dst *QuotaState, cooldown QuotaState) {
dst.Reason = cooldown.Reason
dst.NextRecoverAt = cooldown.NextRecoverAt
dst.BackoffLevel = cooldown.BackoffLevel
dst.cooldownSignals = maps.Clone(cooldown.cooldownSignals)
dst.cooldownEvaluated = cooldown.Exceeded || cooldown.cooldownEvaluated
}
func rejectedClaudeSignalsForCooldown(provider string, headers http.Header) map[string]string {
if !strings.EqualFold(strings.TrimSpace(provider), "claude") {
return nil
}
signals := collectQuotaSignals("claude", headers)
if !hasRejectedClaudeUnifiedSignals(signals) {
return nil
}
return signals
}
func sanitizePersistedClaudeCooldownSignals(raw map[string]string) map[string]string {
if len(raw) == 0 {
return nil
}
headers := make(http.Header, len(raw))
for key, val := range raw {
if strings.HasPrefix(strings.ToLower(strings.TrimSpace(key)), "anthropic-ratelimit-unified-") {
headers.Set(key, val)
}
}
return rejectedClaudeSignalsForCooldown("claude", headers)
}
// collectQuotaSignals builds the bounded snapshot for a single response.

View File

@@ -682,6 +682,11 @@ func (s *authScheduler) mixedUnavailableErrorLocked(providers []string, model st
var latestAuthID string
var latestAuthErr error
allClaude := true
var earliestClaudeTime time.Time
var earliestClaudeID string
var earliestClaudeSignals map[string]string
for _, providerKey := range providers {
providerState := s.providers[providerKey]
if providerState == nil {
@@ -698,6 +703,18 @@ func (s *authScheduler) mixedUnavailableErrorLocked(providers []string, model st
if !localEarliest.IsZero() && (earliest.IsZero() || localEarliest.Before(earliest)) {
earliest = localEarliest
}
if localTotal > 0 {
localClaude, localClaudeEarliest, localClaudeID, localSignals := shard.claudeCooldownSummaryLocked(providerKey, model, now, predicate)
if !localClaude {
allClaude = false
} else if allClaude && !localClaudeEarliest.IsZero() {
if earliestClaudeTime.IsZero() || localClaudeEarliest.Before(earliestClaudeTime) || (localClaudeEarliest.Equal(earliestClaudeTime) && localClaudeID < earliestClaudeID) {
earliestClaudeTime = localClaudeEarliest
earliestClaudeID = localClaudeID
earliestClaudeSignals = localSignals
}
}
}
candModelErr, candModelTime, candModelAuthID, candAuthErr, candAuthTime, candAuthAuthID := shard.candidateErrorsLocked(model, predicate)
if candModelErr != nil {
if latestModelErr == nil || candModelTime.After(latestModelTime) || (candModelTime.Equal(latestModelTime) && candModelAuthID > latestModelAuthID) {
@@ -730,7 +747,11 @@ func (s *authScheduler) mixedUnavailableErrorLocked(providers []string, model st
if resetIn < 0 {
resetIn = 0
}
return newModelCooldownErrorWithCause(model, "", resetIn, lastCandidateErr)
var claudeSignals map[string]string
if allClaude {
claudeSignals = earliestClaudeSignals
}
return newModelCooldownErrorForCandidates(model, "", resetIn, lastCandidateErr, allClaude, claudeSignals)
}
if unauthorizedCount == total {
terminalCause := latestAuthErr
@@ -1475,7 +1496,8 @@ func (m *modelScheduler) unavailableErrorLocked(provider, model string, predicat
if resetIn < 0 {
resetIn = 0
}
return newModelCooldownErrorWithCause(model, providerForError, resetIn, lastCandidateErr)
claudeCooldown, _, _, claudeSignals := m.claudeCooldownSummaryLocked(provider, model, now, predicate)
return newModelCooldownErrorForCandidates(model, providerForError, resetIn, lastCandidateErr, claudeCooldown, claudeSignals)
}
if unauthorizedCount == total {
terminalCause := authErr
@@ -1591,6 +1613,57 @@ func (m *modelScheduler) availabilitySummaryLocked(predicate func(*scheduledAuth
return total, cooldownCount, unauthorizedCount, earliest
}
func (m *modelScheduler) claudeCooldownSummaryLocked(fallbackProvider, model string, now time.Time, predicate func(*scheduledAuth) bool) (bool, time.Time, string, map[string]string) {
if m == nil {
return false, time.Time{}, "", nil
}
matched := 0
var earliest time.Time
var earliestID string
var earliestAuth *Auth
var earliestSignals map[string]string
for _, entry := range m.entries {
if predicate != nil && !predicate(entry) {
continue
}
matched++
if entry == nil || entry.auth == nil {
return false, time.Time{}, "", nil
}
entryProvider := ""
if entry.meta != nil && entry.meta.providerKey != "" {
entryProvider = entry.meta.providerKey
} else if resolved := executorKeyFromAuth(entry.auth); resolved != "" {
entryProvider = resolved
} else {
entryProvider = strings.ToLower(strings.TrimSpace(fallbackProvider))
}
if entryProvider != "claude" {
return false, time.Time{}, "", nil
}
checkModel := model
if existingModelState(entry.auth, checkModel) == nil && m.modelKey != "" {
checkModel = m.modelKey
}
okClaude, signals := claudeUnifiedCooldownForAuth(entry.auth, checkModel, now)
if !okClaude {
return false, time.Time{}, "", nil
}
if entry.state == scheduledStateCooldown && !entry.nextRetryAt.IsZero() {
if earliestAuth == nil || entry.nextRetryAt.Before(earliest) || (entry.nextRetryAt.Equal(earliest) && entry.auth.ID < earliestID) {
earliest = entry.nextRetryAt
earliestID = entry.auth.ID
earliestAuth = entry.auth
earliestSignals = signals
}
}
}
if matched == 0 || earliestAuth == nil {
return false, time.Time{}, "", nil
}
return true, earliest, earliestID, earliestSignals
}
// rebuildIndexesLocked reconstructs ready and blocked views from the current entry map.
func (m *modelScheduler) rebuildIndexesLocked() {
cursorStates := make(map[int]readyBucketCursorState, len(m.readyByPriority))

View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"hash/fnv"
"maps"
"math"
"net/http"
"regexp"
@@ -83,10 +84,12 @@ const (
)
type modelCooldownError struct {
model string
resetIn time.Duration
provider string
cause error
model string
resetIn time.Duration
provider string
cause error
claudeCooldown bool
claudeSignals map[string]string
}
// NewModelCooldownError creates an error representing model-level cooldown.
@@ -99,17 +102,27 @@ func newModelCooldownError(model, provider string, resetIn time.Duration) *model
}
func newModelCooldownErrorWithCause(model, provider string, resetIn time.Duration, cause error) *modelCooldownError {
return newModelCooldownErrorForCandidates(model, provider, resetIn, cause, isClaudeCooldownProvider(provider), nil)
}
func newModelCooldownErrorForCandidates(model, provider string, resetIn time.Duration, cause error, claudeCooldown bool, claudeSignals map[string]string) *modelCooldownError {
if resetIn < 0 {
resetIn = 0
}
return &modelCooldownError{
model: model,
provider: provider,
resetIn: resetIn,
cause: cause,
model: model,
provider: provider,
resetIn: resetIn,
cause: cause,
claudeCooldown: claudeCooldown,
claudeSignals: maps.Clone(claudeSignals),
}
}
func isClaudeCooldownProvider(provider string) bool {
return strings.EqualFold(strings.TrimSpace(provider), "claude")
}
func (e *modelCooldownError) IsModelCooldown() bool {
return true
}
@@ -352,17 +365,419 @@ func (e *modelCooldownError) StatusCode() int {
return http.StatusTooManyRequests
}
func (e *modelCooldownError) Headers() http.Header {
func (e *modelCooldownError) resetSeconds() int64 {
if e == nil || e.resetIn <= 0 {
return 0
}
seconds := int64(e.resetIn / time.Second)
if e.resetIn%time.Second > 0 {
seconds++
}
return seconds
}
// SafeResponseHeaders exposes only the locally trusted cooldown headers.
func (e *modelCooldownError) SafeResponseHeaders() http.Header {
headers := make(http.Header)
headers.Set("Content-Type", "application/json")
resetSeconds := int(math.Ceil(e.resetIn.Seconds()))
if resetSeconds < 0 {
resetSeconds = 0
}
headers.Set("Retry-After", strconv.Itoa(resetSeconds))
headers.Set("Retry-After", strconv.FormatInt(e.resetSeconds(), 10))
return headers
}
func (e *modelCooldownError) Headers() http.Header {
headers := e.SafeResponseHeaders()
if e != nil && e.claudeCooldown && e.resetIn > 0 {
e.appendClaudeUnifiedHeaders(headers)
}
return headers
}
func (e *modelCooldownError) appendClaudeUnifiedHeaders(headers http.Header) {
resetSeconds := e.resetSeconds()
if resetSeconds <= 0 {
return
}
resetAt := time.Now().Unix() + resetSeconds
resetStr := strconv.FormatInt(resetAt, 10)
if hasRejectedClaudeUnifiedSignals(e.claudeSignals) {
for key, value := range e.claudeSignals {
if strings.HasPrefix(strings.ToLower(strings.TrimSpace(key)), "anthropic-ratelimit-unified-") {
headers.Set(key, value)
}
}
headers.Set("Anthropic-Ratelimit-Unified-Status", "rejected")
headers.Set("Anthropic-Ratelimit-Unified-Reset", resetStr)
rejWindows := rejectedClaudeUnifiedWindows(headers)
claim := strings.ToLower(strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-Representative-Claim")))
if len(rejWindows) == 0 && claim == "" && isClaudeOverageOnlyHeaders(headers) {
claim = "overage"
headers.Set("Anthropic-Ratelimit-Unified-Representative-Claim", claim)
}
if len(rejWindows) == 0 && claim != "overage" {
fallbackWindow := fallbackClaudeWindowForClaim(headers, claim, e.resetIn)
if fallbackWindow != "" {
statusHeader := claudeUnifiedWindowHeader(fallbackWindow, "Status")
if strings.TrimSpace(headers.Get(statusHeader)) == "" {
headers.Set(statusHeader, "rejected")
rejWindows = append(rejWindows, fallbackWindow)
}
}
}
if len(rejWindows) == 1 {
headers.Set(claudeUnifiedWindowHeader(rejWindows[0], "Reset"), resetStr)
} else if len(rejWindows) > 1 {
latestHeader := ""
var latestEpoch int64
for _, window := range rejWindows {
resetHeader := claudeUnifiedWindowHeader(window, "Reset")
raw := strings.TrimSpace(headers.Get(resetHeader))
if raw == "" {
headers.Set(resetHeader, resetStr)
continue
}
if parsed, errParse := strconv.ParseInt(raw, 10, 64); errParse == nil && (latestHeader == "" || parsed >= latestEpoch) {
latestHeader = resetHeader
latestEpoch = parsed
}
}
if latestHeader != "" {
headers.Set(latestHeader, resetStr)
}
}
if claim == "" {
if inferred := inferClaudeRepresentativeClaim(headers, rejWindows, e.resetIn); inferred != "" {
headers.Set("Anthropic-Ratelimit-Unified-Representative-Claim", inferred)
}
}
if strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-Overage-Status")) == "" {
headers.Set("Anthropic-Ratelimit-Unified-Overage-Status", "rejected")
}
return
}
claim := "five_hour"
if e.resetIn > 5*time.Hour {
claim = "seven_day"
}
headers.Set("Anthropic-Ratelimit-Unified-Status", "rejected")
headers.Set("Anthropic-Ratelimit-Unified-Reset", resetStr)
headers.Set("Anthropic-Ratelimit-Unified-Representative-Claim", claim)
headers.Set("Anthropic-Ratelimit-Unified-Overage-Status", "rejected")
if claim == "five_hour" {
headers.Set("Anthropic-Ratelimit-Unified-5h-Status", "rejected")
headers.Set("Anthropic-Ratelimit-Unified-5h-Reset", resetStr)
} else {
headers.Set("Anthropic-Ratelimit-Unified-7d-Status", "rejected")
headers.Set("Anthropic-Ratelimit-Unified-7d-Reset", resetStr)
}
}
func claudeUnifiedWindowName(headerKey string) (string, bool) {
lower := strings.ToLower(strings.TrimSpace(headerKey))
rest, hasPrefix := strings.CutPrefix(lower, "anthropic-ratelimit-unified-")
if !hasPrefix {
return "", false
}
window, hasSuffix := strings.CutSuffix(rest, "-status")
if !hasSuffix || window == "" || window == "overage" {
return "", false
}
return window, true
}
func claudeUnifiedWindowHeader(window, field string) string {
return http.CanonicalHeaderKey("Anthropic-Ratelimit-Unified-" + window + "-" + field)
}
func rejectedClaudeUnifiedWindows(headers http.Header) []string {
var windows []string
for key := range headers {
window, ok := claudeUnifiedWindowName(key)
if ok && strings.EqualFold(strings.TrimSpace(headers.Get(key)), "rejected") {
windows = append(windows, window)
}
}
sort.Strings(windows)
return windows
}
func isAllowedClaudeWindowStatus(status string) bool {
lower := strings.ToLower(strings.TrimSpace(status))
return lower == "allowed" || lower == "allowed_warning"
}
func isClaudeUtilizationBelowCap(raw string) bool {
raw = strings.TrimSpace(raw)
if raw == "" {
return false
}
u, errParse := strconv.ParseFloat(raw, 64)
if errParse != nil || math.IsNaN(u) || math.IsInf(u, 0) {
return false
}
return u >= 0 && u < 1.0
}
func isClaudeOverageOnlyHeaders(headers http.Header) bool {
overageRejected := strings.EqualFold(strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-Overage-Status")), "rejected") ||
strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-Overage-Disabled-Reason")) != ""
if !overageRejected {
return false
}
status5h := strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-5h-Status"))
status7d := strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-7d-Status"))
allowed5h := isAllowedClaudeWindowStatus(status5h)
allowed7d := isAllowedClaudeWindowStatus(status7d)
if allowed5h && allowed7d {
return true
}
if allowed7d && status5h == "" {
util5h := strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-5h-Utilization"))
if util5h == "" || isClaudeUtilizationBelowCap(util5h) {
return true
}
}
if allowed5h && status7d == "" && isClaudeUtilizationBelowCap(headers.Get("Anthropic-Ratelimit-Unified-7d-Utilization")) {
return true
}
return false
}
func fallbackClaudeWindowForClaim(headers http.Header, claim string, resetIn time.Duration) string {
switch {
case claim == "seven_day_overage_included":
return "7d_oi"
case strings.HasPrefix(claim, "seven_day_"):
if suffix := strings.TrimPrefix(claim, "seven_day_"); suffix != "" {
return "7d_" + suffix
}
case claim == "seven_day":
return "7d"
case claim == "five_hour":
return "5h"
case claim == "":
allowed5h := isAllowedClaudeWindowStatus(headers.Get("Anthropic-Ratelimit-Unified-5h-Status"))
allowed7d := isAllowedClaudeWindowStatus(headers.Get("Anthropic-Ratelimit-Unified-7d-Status"))
switch {
case allowed5h && allowed7d:
return ""
case allowed5h:
return "7d"
case allowed7d:
return "5h"
case resetIn > 5*time.Hour:
return "7d"
default:
return "5h"
}
}
return ""
}
func inferClaudeRepresentativeClaim(headers http.Header, windows []string, resetIn time.Duration) string {
has5h := false
has7d := false
has7dOI := false
modelFamily7d := ""
otherWindow := ""
for _, window := range windows {
switch {
case window == "7d_oi":
has7dOI = true
case strings.HasPrefix(window, "7d_"):
if modelFamily7d == "" {
modelFamily7d = "seven_day_" + strings.TrimPrefix(window, "7d_")
}
case window == "7d":
has7d = true
case window == "5h":
has5h = true
default:
if otherWindow == "" {
otherWindow = window
}
}
}
switch {
case has7dOI:
return "seven_day_overage_included"
case modelFamily7d != "":
return modelFamily7d
case has7d:
return "seven_day"
case has5h:
return "five_hour"
case otherWindow != "":
return otherWindow
}
if strings.EqualFold(strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-Overage-Status")), "rejected") ||
strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-Overage-Disabled-Reason")) != "" {
return "overage"
}
allowed5h := isAllowedClaudeWindowStatus(headers.Get("Anthropic-Ratelimit-Unified-5h-Status"))
allowed7d := isAllowedClaudeWindowStatus(headers.Get("Anthropic-Ratelimit-Unified-7d-Status"))
switch {
case allowed5h && allowed7d:
return ""
case allowed5h:
return "seven_day"
case allowed7d:
return "five_hour"
case resetIn > 5*time.Hour:
return "seven_day"
default:
return "five_hour"
}
}
func hasRejectedClaudeUnifiedSignals(signals map[string]string) bool {
if len(signals) == 0 {
return false
}
if strings.EqualFold(strings.TrimSpace(signals["Anthropic-Ratelimit-Unified-Status"]), "rejected") {
return true
}
for key, val := range signals {
if _, ok := claudeUnifiedWindowName(key); ok && strings.EqualFold(strings.TrimSpace(val), "rejected") {
return true
}
}
return false
}
func claudeSignalsFromEvaluatedQuota(q QuotaState) (bool, map[string]string) {
if hasRejectedClaudeUnifiedSignals(q.cooldownSignals) {
return true, maps.Clone(q.cooldownSignals)
}
if q.cooldownEvaluated {
return false, nil
}
if hasRejectedClaudeUnifiedSignals(q.Signals) {
return true, maps.Clone(q.Signals)
}
return false, nil
}
func claudeUnifiedCooldownForAuth(auth *Auth, model string, now time.Time) (bool, map[string]string) {
if auth == nil || executorKeyFromAuth(auth) != "claude" {
return false, nil
}
if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now) {
state := existingModelState(auth, model)
if state != nil && state.NextRetryAfter.After(auth.Quota.NextRecoverAt) {
if shadow := state.credentialQuotaShadow; shadow != nil {
if !shadow.Quota.Exceeded || !shadow.Quota.NextRecoverAt.After(now) || shadow.NextRetryAfter.After(shadow.Quota.NextRecoverAt) {
return false, nil
}
if strings.EqualFold(strings.TrimSpace(shadow.Quota.Reason), "cloudflare challenge") {
return false, nil
}
return claudeSignalsFromEvaluatedQuota(shadow.Quota)
}
return false, nil
}
if ok, signals := claudeSignalsFromEvaluatedQuota(auth.Quota); ok {
return true, signals
}
if state != nil && !state.Quota.NextRecoverAt.Before(auth.Quota.NextRecoverAt) {
if ok, signals := claudeSignalsFromEvaluatedQuota(state.Quota); ok {
return true, signals
}
}
return true, nil
}
if len(auth.ModelStates) > 0 {
if model == "" {
if !auth.Unavailable {
return false, nil
}
keys := make([]string, 0, len(auth.ModelStates))
for key := range auth.ModelStates {
keys = append(keys, key)
}
sort.Strings(keys)
var earliestRecover time.Time
var earliestSignals map[string]string
found := false
for _, key := range keys {
st := auth.ModelStates[key]
if st != nil && (st.credentialQuotaPropagated || st.Quota.Reason == "credential_quota") {
st = st.credentialQuotaShadow
}
if st == nil || !st.Quota.Exceeded || !st.Quota.NextRecoverAt.After(now) || st.NextRetryAfter.After(st.Quota.NextRecoverAt) {
return false, nil
}
if strings.EqualFold(strings.TrimSpace(st.Quota.Reason), "cloudflare challenge") {
return false, nil
}
okSt, sig := claudeSignalsFromEvaluatedQuota(st.Quota)
if !okSt {
return false, nil
}
if !found || st.Quota.NextRecoverAt.Before(earliestRecover) {
found = true
earliestRecover = st.Quota.NextRecoverAt
earliestSignals = sig
}
}
return found, earliestSignals
}
state := existingModelState(auth, model)
if state != nil && (state.credentialQuotaPropagated || state.Quota.Reason == "credential_quota") {
state = state.credentialQuotaShadow
}
if state == nil || !state.Quota.Exceeded || !state.Quota.NextRecoverAt.After(now) || state.NextRetryAfter.After(state.Quota.NextRecoverAt) {
return false, nil
}
if strings.EqualFold(strings.TrimSpace(state.Quota.Reason), "cloudflare challenge") {
return false, nil
}
return claudeSignalsFromEvaluatedQuota(state.Quota)
}
if !auth.Quota.Exceeded || !auth.Quota.NextRecoverAt.After(now) || auth.NextRetryAfter.After(auth.Quota.NextRecoverAt) {
return false, nil
}
if strings.EqualFold(strings.TrimSpace(auth.Quota.Reason), "cloudflare challenge") {
return false, nil
}
return claudeSignalsFromEvaluatedQuota(auth.Quota)
}
func claudeCooldownMetadataForAuths(ctx context.Context, auths []*Auth, modelForAuth func(*Auth) string, now time.Time, clientHeaders ...http.Header) (bool, map[string]string) {
if len(auths) == 0 {
return false, nil
}
var earliestAuth *Auth
var earliestSignals map[string]string
var earliestNext time.Time
for _, candidate := range auths {
checkModel := ""
if modelForAuth != nil {
checkModel = modelForAuth(candidate)
}
okClaude, signals := claudeUnifiedCooldownForAuth(candidate, checkModel, now)
if !okClaude {
return false, nil
}
blocked, reason, next := isAuthBlockedForModelForRequest(ctx, candidate, checkModel, now, clientHeaders...)
if blocked && reason == blockReasonCooldown && next.After(now) {
if earliestAuth == nil || next.Before(earliestNext) || (next.Equal(earliestNext) && candidate.ID < earliestAuth.ID) {
earliestAuth = candidate
earliestSignals = signals
earliestNext = next
}
}
}
if earliestAuth == nil {
return false, nil
}
return true, earliestSignals
}
func authPriority(auth *Auth) int {
if auth == nil || auth.Attributes == nil {
return 0
@@ -558,7 +973,10 @@ func getAvailableAuthsWithPriorityModeForRequest(ctx context.Context, auths []*A
if resetIn < 0 {
resetIn = 0
}
return nil, newModelCooldownError(model, providerForError, resetIn)
claudeCooldown, claudeSignals := claudeCooldownMetadataForAuths(ctx, auths, func(*Auth) string {
return model
}, now, clientHeaders...)
return nil, newModelCooldownErrorForCandidates(model, providerForError, resetIn, nil, claudeCooldown, claudeSignals)
}
return nil, newAuthUnavailableError(earliest, now)
}
@@ -819,7 +1237,23 @@ func isAuthBlockedForModelForRequest(ctx context.Context, auth *Auth, model stri
return true, blockReasonOther, time.Time{}
}
if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now) {
return true, blockReasonCooldown, auth.Quota.NextRecoverAt
credNext := auth.Quota.NextRecoverAt
if model != "" && len(auth.ModelStates) > 0 {
modelKey := canonicalModelLookupKey(model)
for stateModel, state := range auth.ModelStates {
if state == nil || canonicalModelLookupKey(stateModel) != modelKey {
continue
}
if state.Status == StatusDisabled {
return true, blockReasonDisabled, time.Time{}
}
stateBlocked, _, next := availabilityBlock(state.Unavailable, state.Quota.Exceeded, state.NextRetryAfter, state.Quota.NextRecoverAt, now)
if stateBlocked && next.After(credNext) {
credNext = next
}
}
}
return true, blockReasonCooldown, credNext
}
unavailable := codexRefreshUnavailableForRequest(ctx, auth, now, clientHeaders...)
if model != "" {

View File

@@ -355,6 +355,10 @@ type QuotaState struct {
// Cooldown transitions must use applyCooldownFields so they cannot replace
// this snapshot.
Signals map[string]string `json:"signals,omitempty"`
// cooldownSignals retains rejected quota headers captured when the active
// cooldown was recorded so out-of-order completions cannot overwrite them.
cooldownSignals map[string]string
cooldownEvaluated bool
}
// Clone returns an independent copy of the quota state.
@@ -366,6 +370,12 @@ func (q QuotaState) Clone() QuotaState {
copyQuota.Signals[key] = value
}
}
if len(q.cooldownSignals) > 0 {
copyQuota.cooldownSignals = make(map[string]string, len(q.cooldownSignals))
for key, value := range q.cooldownSignals {
copyQuota.cooldownSignals[key] = value
}
}
return copyQuota
}