mirror of
https://github.com/router-for-me/CLIProxyAPI.git
synced 2026-11-16 22:57:53 +08:00
fix(claude): emit anthropic-ratelimit-unified headers on local cooldown 429s
- Retain and persist rejected Claude unified quota signals alongside active cooldown deadlines so out-of-order completions, restarts, and dual-layer quota wraps keep accurate window metadata. - Populate anthropic-ratelimit-unified-* headers on Claude quota modelCooldownError responses aligned with Retry-After while keeping SafeResponseHeaders restricted to trusted retry headers. - Exclude non-Claude, Cloudflare challenge, and ordinary per-minute rate-limit cooldowns from unified subscription quota header synthesis across selector and scheduler paths. Closes: #7417 Co-authored-by: Dylan <1990016@gmail.com> Co-authored-by: lzt404 <2596933790@qq.com>
This commit is contained in:
@@ -224,3 +224,101 @@ func TestClaudeCooldownMessagesRoute(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type cooldownClaudeTestExecutor struct {
|
||||
preFTKeepAliveClaudeExecutor
|
||||
}
|
||||
|
||||
func (*cooldownClaudeTestExecutor) Identifier() string { return "claude" }
|
||||
|
||||
func TestClaudeCooldownMessagesRoute_PassthroughUnifiedRateLimitHeaders(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
chunks := make(chan coreexecutor.StreamChunk)
|
||||
close(chunks)
|
||||
executor := &cooldownClaudeTestExecutor{preFTKeepAliveClaudeExecutor: preFTKeepAliveClaudeExecutor{chunks: chunks}}
|
||||
manager := coreauth.NewManager(nil, coreauth.NewSessionAffinitySelector(&coreauth.RoundRobinSelector{}), nil)
|
||||
manager.RegisterExecutor(executor)
|
||||
const model = "issue-7417-cooldown-route-model"
|
||||
const authID = "issue-7417-cooldown-route-auth"
|
||||
next := time.Now().Add(7823 * time.Second)
|
||||
auth := &coreauth.Auth{
|
||||
ID: authID,
|
||||
Provider: executor.Identifier(),
|
||||
Status: coreauth.StatusActive,
|
||||
Unavailable: true,
|
||||
NextRetryAfter: next,
|
||||
Quota: coreauth.QuotaState{
|
||||
Exceeded: true,
|
||||
Reason: "credential_quota",
|
||||
NextRecoverAt: next,
|
||||
Signals: map[string]string{
|
||||
"Anthropic-Ratelimit-Unified-Status": "rejected",
|
||||
"Anthropic-Ratelimit-Unified-5h-Status": "rejected",
|
||||
"Anthropic-Ratelimit-Unified-Reset": strconv.FormatInt(next.Add(-10*time.Second).Unix(), 10),
|
||||
},
|
||||
},
|
||||
LastError: &coreauth.Error{Code: "rate_limit_error", Message: "upstream quota exceeded", HTTPStatus: http.StatusTooManyRequests},
|
||||
}
|
||||
if _, errRegister := manager.Register(context.Background(), auth); errRegister != nil {
|
||||
t.Fatalf("register auth: %v", errRegister)
|
||||
}
|
||||
registry.GetGlobalRegistry().RegisterClient(authID, executor.Identifier(), []*registry.ModelInfo{{ID: model}})
|
||||
t.Cleanup(func() {
|
||||
registry.GetGlobalRegistry().UnregisterClient(authID)
|
||||
})
|
||||
|
||||
for _, passthrough := range []bool{false, true} {
|
||||
for _, streaming := range []bool{false, true} {
|
||||
name := "passthrough_" + strconv.FormatBool(passthrough) + "_stream_" + strconv.FormatBool(streaming)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
handler := NewClaudeCodeAPIHandler(handlers.NewBaseAPIHandlers(&sdkconfig.SDKConfig{PassthroughHeaders: passthrough}, manager))
|
||||
router := gin.New()
|
||||
router.POST("/v1/messages", handler.ClaudeMessages)
|
||||
|
||||
requestBody := `{"model":"` + model + `","messages":[{"role":"user","content":"hi"}],"stream":` + strconv.FormatBool(streaming) + `}`
|
||||
request := httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(requestBody))
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
recorder := httptest.NewRecorder()
|
||||
|
||||
before := time.Now().Unix()
|
||||
router.ServeHTTP(recorder, request)
|
||||
after := time.Now().Unix()
|
||||
|
||||
if recorder.Code != http.StatusTooManyRequests {
|
||||
t.Fatalf("status = %d, want 429; body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
seconds, errSeconds := strconv.ParseInt(recorder.Header().Get("Retry-After"), 10, 64)
|
||||
if errSeconds != nil || seconds <= 0 {
|
||||
t.Fatalf("Retry-After = %q, want positive seconds", recorder.Header().Get("Retry-After"))
|
||||
}
|
||||
|
||||
if !passthrough {
|
||||
if got := recorder.Header().Get("Anthropic-Ratelimit-Unified-Status"); got != "" {
|
||||
t.Fatalf("passthrough=false leaked Anthropic-Ratelimit-Unified-Status = %q", got)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if got := recorder.Header().Get("Anthropic-Ratelimit-Unified-Status"); got != "rejected" {
|
||||
t.Fatalf("Anthropic-Ratelimit-Unified-Status = %q, want rejected", got)
|
||||
}
|
||||
if got := recorder.Header().Get("Anthropic-Ratelimit-Unified-Representative-Claim"); got != "five_hour" {
|
||||
t.Fatalf("Anthropic-Ratelimit-Unified-Representative-Claim = %q, want five_hour", got)
|
||||
}
|
||||
if got := recorder.Header().Get("Anthropic-Ratelimit-Unified-Overage-Status"); got != "rejected" {
|
||||
t.Fatalf("Anthropic-Ratelimit-Unified-Overage-Status = %q, want rejected", got)
|
||||
}
|
||||
if got := recorder.Header().Get("Anthropic-Ratelimit-Unified-5h-Status"); got != "rejected" {
|
||||
t.Fatalf("Anthropic-Ratelimit-Unified-5h-Status = %q, want rejected", got)
|
||||
}
|
||||
resetAt, errReset := strconv.ParseInt(recorder.Header().Get("Anthropic-Ratelimit-Unified-Reset"), 10, 64)
|
||||
if errReset != nil || resetAt < before+seconds || resetAt > after+seconds {
|
||||
t.Fatalf("Anthropic-Ratelimit-Unified-Reset = %q, want unix epoch in [%d, %d]", recorder.Header().Get("Anthropic-Ratelimit-Unified-Reset"), before+seconds, after+seconds)
|
||||
}
|
||||
if got := recorder.Header().Get("Anthropic-Ratelimit-Unified-5h-Reset"); got != strconv.FormatInt(resetAt, 10) {
|
||||
t.Fatalf("Anthropic-Ratelimit-Unified-5h-Reset = %q, want %d", got, resetAt)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net"
|
||||
"net/http"
|
||||
"regexp"
|
||||
@@ -382,11 +383,18 @@ func (m *Manager) restoreCooldownRecordLocked(record CooldownStateRecord, now ti
|
||||
if quota.Exceeded && quota.NextRecoverAt.IsZero() {
|
||||
quota.NextRecoverAt = record.NextRetryAfter
|
||||
}
|
||||
quota.cooldownEvaluated = quota.Exceeded
|
||||
if executorKeyFromAuth(auth) == "claude" || strings.EqualFold(strings.TrimSpace(record.Provider), "claude") {
|
||||
quota.cooldownSignals = sanitizePersistedClaudeCooldownSignals(record.CooldownSignals)
|
||||
}
|
||||
|
||||
if model == "" {
|
||||
auth.Unavailable = true
|
||||
auth.Status = StatusError
|
||||
auth.NextRetryAfter = record.NextRetryAfter
|
||||
if len(quota.cooldownSignals) == 0 && auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(now) && !quota.NextRecoverAt.After(auth.Quota.NextRecoverAt) {
|
||||
quota.cooldownSignals = auth.Quota.cooldownSignals
|
||||
}
|
||||
applyCooldownFields(&auth.Quota, quota)
|
||||
auth.Quota = mergeQuotaObservation(auth.Quota, quota)
|
||||
auth.Generation++
|
||||
@@ -453,10 +461,11 @@ func (m *Manager) restoreCooldownRecordLocked(record CooldownStateRecord, now ti
|
||||
}
|
||||
state.credentialQuotaPropagated = true
|
||||
applyCooldownFields(&state.Quota, QuotaState{
|
||||
Exceeded: true,
|
||||
Reason: "credential_quota",
|
||||
NextRecoverAt: auth.Quota.NextRecoverAt,
|
||||
BackoffLevel: auth.Quota.BackoffLevel,
|
||||
Exceeded: true,
|
||||
Reason: "credential_quota",
|
||||
NextRecoverAt: auth.Quota.NextRecoverAt,
|
||||
BackoffLevel: auth.Quota.BackoffLevel,
|
||||
cooldownSignals: auth.Quota.cooldownSignals,
|
||||
})
|
||||
if state.NextRetryAfter.Before(auth.Quota.NextRecoverAt) {
|
||||
state.NextRetryAfter = auth.Quota.NextRecoverAt
|
||||
@@ -466,7 +475,12 @@ func (m *Manager) restoreCooldownRecordLocked(record CooldownStateRecord, now ti
|
||||
} else if isWrappedRecord {
|
||||
state.credentialQuotaPropagated = true
|
||||
if record.IndependentState != nil && (liveShadow == nil || !liveShadow.UpdatedAt.After(record.IndependentState.UpdatedAt)) {
|
||||
state.credentialQuotaShadow = record.IndependentState.cloneShallow()
|
||||
restoredShadow := record.IndependentState.cloneShallow()
|
||||
restoredShadow.Quota.cooldownEvaluated = restoredShadow.Quota.Exceeded
|
||||
if executorKeyFromAuth(auth) == "claude" || strings.EqualFold(strings.TrimSpace(record.Provider), "claude") {
|
||||
restoredShadow.Quota.cooldownSignals = sanitizePersistedClaudeCooldownSignals(record.IndependentCooldownSignals)
|
||||
}
|
||||
state.credentialQuotaShadow = restoredShadow
|
||||
} else if liveShadow != nil {
|
||||
state.credentialQuotaShadow = liveShadow
|
||||
}
|
||||
@@ -756,6 +770,9 @@ func cooldownStateRecordEqual(a, b CooldownStateRecord) bool {
|
||||
if !cooldownErrorEqual(a.LastError, b.LastError) {
|
||||
return false
|
||||
}
|
||||
if !maps.Equal(a.CooldownSignals, b.CooldownSignals) || !maps.Equal(a.IndependentCooldownSignals, b.IndependentCooldownSignals) {
|
||||
return false
|
||||
}
|
||||
return cooldownIndependentStateEqual(a.IndependentState, b.IndependentState)
|
||||
}
|
||||
|
||||
@@ -793,16 +810,21 @@ func authCooldownStateRecord(auth *Auth, now time.Time) (CooldownStateRecord, bo
|
||||
if auth == nil || !auth.Unavailable || auth.NextRetryAfter.IsZero() || !auth.NextRetryAfter.After(now) {
|
||||
return CooldownStateRecord{}, false
|
||||
}
|
||||
var cooldownSignals map[string]string
|
||||
if auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(now) && !auth.NextRetryAfter.After(auth.Quota.NextRecoverAt) {
|
||||
cooldownSignals = sanitizePersistedClaudeCooldownSignals(auth.Quota.cooldownSignals)
|
||||
}
|
||||
return CooldownStateRecord{
|
||||
Provider: strings.TrimSpace(auth.Provider),
|
||||
AuthID: auth.ID,
|
||||
AuthFile: cooldownAuthFile(auth),
|
||||
Status: "cooling",
|
||||
NextRetryAfter: auth.NextRetryAfter,
|
||||
Reason: cooldownReason(auth.StatusMessage, auth.Quota, auth.LastError),
|
||||
Quota: cooldownFieldsOf(auth.Quota),
|
||||
LastError: cloneError(auth.LastError),
|
||||
UpdatedAt: auth.UpdatedAt,
|
||||
Provider: strings.TrimSpace(auth.Provider),
|
||||
AuthID: auth.ID,
|
||||
AuthFile: cooldownAuthFile(auth),
|
||||
Status: "cooling",
|
||||
NextRetryAfter: auth.NextRetryAfter,
|
||||
Reason: cooldownReason(auth.StatusMessage, auth.Quota, auth.LastError),
|
||||
Quota: cooldownFieldsOf(auth.Quota),
|
||||
LastError: cloneError(auth.LastError),
|
||||
UpdatedAt: auth.UpdatedAt,
|
||||
CooldownSignals: cooldownSignals,
|
||||
}, true
|
||||
}
|
||||
|
||||
@@ -811,20 +833,32 @@ func modelCooldownStateRecord(auth *Auth, model string, state *ModelState, now t
|
||||
if auth == nil || state == nil || model == "" || !state.Unavailable || state.NextRetryAfter.IsZero() || !state.NextRetryAfter.After(now) {
|
||||
return CooldownStateRecord{}, false
|
||||
}
|
||||
var cooldownSignals map[string]string
|
||||
liveCredentialWrap := auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now)
|
||||
expiredCredentialWrap := (state.credentialQuotaPropagated || state.Quota.Reason == "credential_quota") && !liveCredentialWrap
|
||||
if !expiredCredentialWrap && state.Quota.Exceeded && state.Quota.NextRecoverAt.After(now) &&
|
||||
(state.Quota.Reason == "credential_quota" || !state.NextRetryAfter.After(state.Quota.NextRecoverAt)) {
|
||||
cooldownSignals = sanitizePersistedClaudeCooldownSignals(state.Quota.cooldownSignals)
|
||||
}
|
||||
record := CooldownStateRecord{
|
||||
Provider: strings.TrimSpace(auth.Provider),
|
||||
AuthID: auth.ID,
|
||||
AuthFile: cooldownAuthFile(auth),
|
||||
Model: model,
|
||||
Status: "cooling",
|
||||
NextRetryAfter: state.NextRetryAfter,
|
||||
Reason: cooldownReason(state.StatusMessage, state.Quota, state.LastError),
|
||||
Quota: cooldownFieldsOf(state.Quota),
|
||||
LastError: cloneError(state.LastError),
|
||||
UpdatedAt: state.UpdatedAt,
|
||||
Provider: strings.TrimSpace(auth.Provider),
|
||||
AuthID: auth.ID,
|
||||
AuthFile: cooldownAuthFile(auth),
|
||||
Model: model,
|
||||
Status: "cooling",
|
||||
NextRetryAfter: state.NextRetryAfter,
|
||||
Reason: cooldownReason(state.StatusMessage, state.Quota, state.LastError),
|
||||
Quota: cooldownFieldsOf(state.Quota),
|
||||
LastError: cloneError(state.LastError),
|
||||
UpdatedAt: state.UpdatedAt,
|
||||
CooldownSignals: cooldownSignals,
|
||||
}
|
||||
if state.credentialQuotaShadow != nil {
|
||||
record.IndependentState = state.credentialQuotaShadow.cloneShallow()
|
||||
if state.credentialQuotaShadow.Quota.Exceeded && state.credentialQuotaShadow.Quota.NextRecoverAt.After(now) &&
|
||||
!state.credentialQuotaShadow.NextRetryAfter.After(state.credentialQuotaShadow.Quota.NextRecoverAt) {
|
||||
record.IndependentCooldownSignals = sanitizePersistedClaudeCooldownSignals(state.credentialQuotaShadow.Quota.cooldownSignals)
|
||||
}
|
||||
}
|
||||
record.CredentialQuotaPropagated = state.credentialQuotaPropagated
|
||||
return record, true
|
||||
@@ -1123,8 +1157,12 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
|
||||
state.NextRetryAfter = next
|
||||
}
|
||||
case 429:
|
||||
prevStateQuotaActive := state.Quota.Exceeded && state.Quota.NextRecoverAt.After(now)
|
||||
prevStateRecoverAt := state.Quota.NextRecoverAt
|
||||
prevStateCooldownSignals := maps.Clone(state.Quota.cooldownSignals)
|
||||
var next time.Time
|
||||
var credentialNext time.Time
|
||||
keptPriorStateQuota := false
|
||||
backoffLevel := state.Quota.BackoffLevel
|
||||
if result.CredentialScope {
|
||||
backoffLevel = 0
|
||||
@@ -1152,16 +1190,32 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
|
||||
next, backoffLevel = quotaCooldownAfterFailure(quotaForFailure, now)
|
||||
}
|
||||
credentialNext = next
|
||||
if state.Quota.Exceeded && state.Quota.NextRecoverAt.After(next) {
|
||||
next = state.Quota.NextRecoverAt
|
||||
if prevStateQuotaActive && (prevStateRecoverAt.After(next) || (!result.CredentialScope && result.RetryAfter == nil)) {
|
||||
next = prevStateRecoverAt
|
||||
keptPriorStateQuota = true
|
||||
} else if result.CredentialScope && result.RetryAfter == nil && auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now) {
|
||||
// quotaCooldownAfterFailure reused the open credential_quota window.
|
||||
keptPriorStateQuota = prevStateQuotaActive && prevStateRecoverAt.Equal(next)
|
||||
}
|
||||
}
|
||||
var cooldownSignals map[string]string
|
||||
if !disableCooling {
|
||||
if keptPriorStateQuota {
|
||||
cooldownSignals = prevStateCooldownSignals
|
||||
} else {
|
||||
cooldownSignals = rejectedClaudeSignalsForCooldown(result.Provider, responseHeaders)
|
||||
if len(cooldownSignals) == 0 && prevStateQuotaActive && prevStateRecoverAt.Equal(next) {
|
||||
cooldownSignals = prevStateCooldownSignals
|
||||
}
|
||||
}
|
||||
}
|
||||
state.NextRetryAfter = next
|
||||
applyCooldownFields(&state.Quota, QuotaState{
|
||||
Exceeded: true,
|
||||
Reason: "quota",
|
||||
NextRecoverAt: next,
|
||||
BackoffLevel: backoffLevel,
|
||||
Exceeded: true,
|
||||
Reason: "quota",
|
||||
NextRecoverAt: next,
|
||||
BackoffLevel: backoffLevel,
|
||||
cooldownSignals: cooldownSignals,
|
||||
})
|
||||
// Non-credential own failures re-derive their independent layer
|
||||
// below, so a stale shadow is dropped. A credential-scoped
|
||||
@@ -1187,12 +1241,24 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
|
||||
}
|
||||
}
|
||||
if result.CredentialScope && !disableCooling {
|
||||
prevAuthCredActive := auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now)
|
||||
prevAuthCredRecoverAt := auth.Quota.NextRecoverAt
|
||||
prevAuthCredSignals := maps.Clone(auth.Quota.cooldownSignals)
|
||||
var credCooldownSignals map[string]string
|
||||
if prevAuthCredActive && (prevAuthCredRecoverAt.After(credentialNext) || result.RetryAfter == nil) {
|
||||
credCooldownSignals = prevAuthCredSignals
|
||||
} else {
|
||||
credCooldownSignals = rejectedClaudeSignalsForCooldown(result.Provider, responseHeaders)
|
||||
if len(credCooldownSignals) == 0 && prevAuthCredActive && prevAuthCredRecoverAt.Equal(credentialNext) {
|
||||
credCooldownSignals = prevAuthCredSignals
|
||||
}
|
||||
}
|
||||
for _, otherState := range auth.ModelStates {
|
||||
if otherState != nil && otherState != state {
|
||||
// Normalize a naturally expired wrap first (issue #6690) so a
|
||||
// stale marker cannot suppress snapshotting a fresh block.
|
||||
if (otherState.credentialQuotaPropagated || otherState.Quota.Reason == "credential_quota") &&
|
||||
!(auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now)) {
|
||||
!prevAuthCredActive {
|
||||
if shadow := otherState.credentialQuotaShadow; shadow != nil && isModelStateActiveCooldown(shadow, now) {
|
||||
preservedSeq := shadow.cooldownRecordedSeq
|
||||
*otherState = *shadow.cloneShallow()
|
||||
@@ -1219,8 +1285,13 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
|
||||
otherState.Unavailable = true
|
||||
otherState.Status = StatusError
|
||||
otherQuotaNext := credentialNext
|
||||
otherSignals := credCooldownSignals
|
||||
if prevAuthCredActive && prevAuthCredRecoverAt.After(otherQuotaNext) {
|
||||
otherQuotaNext = prevAuthCredRecoverAt
|
||||
}
|
||||
if otherState.Quota.Exceeded && otherState.Quota.NextRecoverAt.After(otherQuotaNext) {
|
||||
otherQuotaNext = otherState.Quota.NextRecoverAt
|
||||
otherSignals = maps.Clone(otherState.Quota.cooldownSignals)
|
||||
}
|
||||
otherRetryAfter := otherQuotaNext
|
||||
// Propagation only extends a sibling's still-live
|
||||
@@ -1230,25 +1301,34 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
|
||||
}
|
||||
otherState.NextRetryAfter = otherRetryAfter
|
||||
applyCooldownFields(&otherState.Quota, QuotaState{
|
||||
Exceeded: true,
|
||||
Reason: "credential_quota",
|
||||
NextRecoverAt: otherQuotaNext,
|
||||
BackoffLevel: backoffLevel,
|
||||
Exceeded: true,
|
||||
Reason: "credential_quota",
|
||||
NextRecoverAt: otherQuotaNext,
|
||||
BackoffLevel: backoffLevel,
|
||||
cooldownSignals: otherSignals,
|
||||
})
|
||||
}
|
||||
}
|
||||
if !wasTerminalUnauthorized {
|
||||
auth.Unavailable = true
|
||||
authNext := credentialNext
|
||||
if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" &&
|
||||
auth.Quota.NextRecoverAt.After(authNext) {
|
||||
authNext = auth.Quota.NextRecoverAt
|
||||
if prevAuthCredActive && prevAuthCredRecoverAt.After(authNext) {
|
||||
authNext = prevAuthCredRecoverAt
|
||||
}
|
||||
auth.Quota.Exceeded = true
|
||||
auth.Quota.Reason = "credential_quota"
|
||||
auth.Quota.NextRecoverAt = authNext
|
||||
auth.Quota.BackoffLevel = backoffLevel
|
||||
auth.Quota.cooldownSignals = maps.Clone(credCooldownSignals)
|
||||
auth.Quota.cooldownEvaluated = true
|
||||
auth.NextRetryAfter = authNext
|
||||
if authNext.After(state.Quota.NextRecoverAt) || (prevAuthCredActive && result.RetryAfter == nil) {
|
||||
state.Quota.NextRecoverAt = authNext
|
||||
state.Quota.cooldownSignals = maps.Clone(credCooldownSignals)
|
||||
if state.NextRetryAfter.Before(authNext) {
|
||||
state.NextRetryAfter = authNext
|
||||
}
|
||||
}
|
||||
// The reporting model is now part of the wrap too: mark it
|
||||
// so a later credential report from another model cannot
|
||||
// snapshot this credential-derived state as independent.
|
||||
@@ -1310,10 +1390,11 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
|
||||
}
|
||||
if !(outer.Quota.Exceeded && outer.Quota.Reason == "credential_quota") {
|
||||
applyCooldownFields(&outer.Quota, QuotaState{
|
||||
Exceeded: true,
|
||||
Reason: "credential_quota",
|
||||
NextRecoverAt: auth.Quota.NextRecoverAt,
|
||||
BackoffLevel: auth.Quota.BackoffLevel,
|
||||
Exceeded: true,
|
||||
Reason: "credential_quota",
|
||||
NextRecoverAt: auth.Quota.NextRecoverAt,
|
||||
BackoffLevel: auth.Quota.BackoffLevel,
|
||||
cooldownSignals: auth.Quota.cooldownSignals,
|
||||
})
|
||||
}
|
||||
*outerState = *outer
|
||||
@@ -1337,7 +1418,36 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
|
||||
disableCooling = false
|
||||
}
|
||||
if !wasTerminalUnauthorized {
|
||||
prevAuthQuotaActive := auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(now)
|
||||
prevAuthRecoverAt := auth.Quota.NextRecoverAt
|
||||
prevAuthCooldownSignals := maps.Clone(auth.Quota.cooldownSignals)
|
||||
keptPriorAuthQuota := false
|
||||
if !disableCooling && statusCodeFromResult(result.Error) == 429 && prevAuthQuotaActive {
|
||||
if result.RetryAfter == nil {
|
||||
keptPriorAuthQuota = true
|
||||
} else {
|
||||
cooldown := *result.RetryAfter
|
||||
if cooldown < minQuotaCooldownFloor {
|
||||
cooldown = minQuotaCooldownFloor
|
||||
}
|
||||
if prevAuthRecoverAt.After(now.Add(cooldown).Round(0)) {
|
||||
keptPriorAuthQuota = true
|
||||
}
|
||||
}
|
||||
}
|
||||
applyAuthFailureState(auth, result.Error, result.RetryAfter, now, disableCooling)
|
||||
if !disableCooling && statusCodeFromResult(result.Error) == 429 && auth.Quota.Exceeded {
|
||||
auth.Quota.cooldownEvaluated = true
|
||||
if keptPriorAuthQuota {
|
||||
auth.Quota.cooldownSignals = prevAuthCooldownSignals
|
||||
} else if rejectedSignals := rejectedClaudeSignalsForCooldown(result.Provider, responseHeaders); len(rejectedSignals) > 0 {
|
||||
auth.Quota.cooldownSignals = rejectedSignals
|
||||
} else if prevAuthQuotaActive && auth.Quota.NextRecoverAt.Equal(prevAuthRecoverAt) {
|
||||
auth.Quota.cooldownSignals = prevAuthCooldownSignals
|
||||
} else {
|
||||
auth.Quota.cooldownSignals = nil
|
||||
}
|
||||
}
|
||||
auth.cooldownRecordedSeq = m.currentExecutionSeq()
|
||||
}
|
||||
}
|
||||
@@ -1630,15 +1740,25 @@ func mergeModelState(target, source *ModelState) *ModelState {
|
||||
NextRetryAfter: target.NextRetryAfter,
|
||||
LastError: cloneError(preferred.LastError),
|
||||
Quota: QuotaState{
|
||||
Exceeded: target.Quota.Exceeded || source.Quota.Exceeded,
|
||||
Reason: preferred.Quota.Reason,
|
||||
NextRecoverAt: target.Quota.NextRecoverAt,
|
||||
BackoffLevel: target.Quota.BackoffLevel,
|
||||
Exceeded: target.Quota.Exceeded || source.Quota.Exceeded,
|
||||
Reason: preferred.Quota.Reason,
|
||||
NextRecoverAt: target.Quota.NextRecoverAt,
|
||||
BackoffLevel: target.Quota.BackoffLevel,
|
||||
cooldownEvaluated: target.Quota.cooldownEvaluated || source.Quota.cooldownEvaluated,
|
||||
},
|
||||
UpdatedAt: target.UpdatedAt,
|
||||
}
|
||||
merged.Quota = mergeQuotaObservation(merged.Quota, fallback.Quota)
|
||||
merged.Quota = mergeQuotaObservation(merged.Quota, preferred.Quota)
|
||||
if source.Quota.NextRecoverAt.After(target.Quota.NextRecoverAt) {
|
||||
merged.Quota.cooldownSignals = maps.Clone(source.Quota.cooldownSignals)
|
||||
} else if target.Quota.NextRecoverAt.After(source.Quota.NextRecoverAt) {
|
||||
merged.Quota.cooldownSignals = maps.Clone(target.Quota.cooldownSignals)
|
||||
} else if len(preferred.Quota.cooldownSignals) > 0 {
|
||||
merged.Quota.cooldownSignals = maps.Clone(preferred.Quota.cooldownSignals)
|
||||
} else if len(fallback.Quota.cooldownSignals) > 0 {
|
||||
merged.Quota.cooldownSignals = maps.Clone(fallback.Quota.cooldownSignals)
|
||||
}
|
||||
if source.NextRetryAfter.After(merged.NextRetryAfter) {
|
||||
merged.NextRetryAfter = source.NextRetryAfter
|
||||
}
|
||||
@@ -1873,10 +1993,7 @@ func updateAggregatedAvailability(auth *Auth, now time.Time) {
|
||||
} else if auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(now) {
|
||||
// Retain active auth-level quota cooldown
|
||||
} else {
|
||||
auth.Quota.Exceeded = false
|
||||
auth.Quota.Reason = ""
|
||||
auth.Quota.NextRecoverAt = time.Time{}
|
||||
auth.Quota.BackoffLevel = 0
|
||||
applyCooldownFields(&auth.Quota, QuotaState{})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1920,10 +2037,7 @@ func clearAuthStateOnSuccess(auth *Auth, now time.Time) {
|
||||
auth.Unavailable = false
|
||||
auth.Status = StatusActive
|
||||
auth.StatusMessage = ""
|
||||
auth.Quota.Exceeded = false
|
||||
auth.Quota.Reason = ""
|
||||
auth.Quota.NextRecoverAt = time.Time{}
|
||||
auth.Quota.BackoffLevel = 0
|
||||
applyCooldownFields(&auth.Quota, QuotaState{})
|
||||
auth.LastError = nil
|
||||
auth.NextRetryAfter = time.Time{}
|
||||
auth.cooldownRecordedSeq = 0
|
||||
|
||||
@@ -909,7 +909,10 @@ func (m *Manager) availableAuthsForRouteModelForRequest(ctx context.Context, aut
|
||||
if resetIn < 0 {
|
||||
resetIn = 0
|
||||
}
|
||||
return nil, newModelCooldownErrorWithCause(routeModel, providerForError, resetIn, lastCandidateErr)
|
||||
claudeCooldown, claudeSignals := claudeCooldownMetadataForAuths(ctx, auths, func(candidate *Auth) string {
|
||||
return m.selectionModelForAuth(candidate, routeModel)
|
||||
}, now, clientHeaders...)
|
||||
return nil, newModelCooldownErrorForCandidates(routeModel, providerForError, resetIn, lastCandidateErr, claudeCooldown, claudeSignals)
|
||||
}
|
||||
if unauthorizedCount == len(auths) && len(auths) > 0 {
|
||||
terminalCause := latestUnauthorizedCandidateError(auths)
|
||||
@@ -1024,7 +1027,14 @@ func restoreModelCooldownErrorModel(err error, requestedModel string) error {
|
||||
if !errors.As(err, &cooldownErr) || cooldownErr == nil || cooldownErr.model != "" {
|
||||
return err
|
||||
}
|
||||
return newModelCooldownErrorWithCause(requestedModel, cooldownErr.provider, cooldownErr.resetIn, cooldownErr.cause)
|
||||
return newModelCooldownErrorForCandidates(
|
||||
requestedModel,
|
||||
cooldownErr.provider,
|
||||
cooldownErr.resetIn,
|
||||
cooldownErr.cause,
|
||||
cooldownErr.claudeCooldown,
|
||||
cooldownErr.claudeSignals,
|
||||
)
|
||||
}
|
||||
|
||||
func latestUnauthorizedCandidateError(auths []*Auth) error {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -34,6 +34,11 @@ type CooldownStateRecord struct {
|
||||
// CredentialQuotaPropagated preserves wrap provenance for models that
|
||||
// carry no independent layer (e.g. the reporting model itself).
|
||||
CredentialQuotaPropagated bool `json:"credential_quota_propagated,omitempty"`
|
||||
// CooldownSignals persists bounded rejected quota headers captured when
|
||||
// this cooldown was recorded without touching passive Quota.Signals watermarks.
|
||||
CooldownSignals map[string]string `json:"cooldown_signals,omitempty"`
|
||||
// IndependentCooldownSignals persists CooldownSignals for IndependentState.
|
||||
IndependentCooldownSignals map[string]string `json:"independent_cooldown_signals,omitempty"`
|
||||
}
|
||||
|
||||
// CooldownStateStore persists runtime cooldown state independently from auth tokens.
|
||||
|
||||
@@ -310,7 +310,7 @@ func SafeResponseHeaders(err error) http.Header {
|
||||
}
|
||||
var modelCooldown *modelCooldownError
|
||||
if errors.As(err, &modelCooldown) && modelCooldown != nil {
|
||||
return modelCooldown.Headers()
|
||||
return modelCooldown.SafeResponseHeaders()
|
||||
}
|
||||
var unavailable *authUnavailableError
|
||||
if errors.As(err, &unavailable) && unavailable != nil {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"maps"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -86,6 +87,32 @@ func applyCooldownFields(dst *QuotaState, cooldown QuotaState) {
|
||||
dst.Reason = cooldown.Reason
|
||||
dst.NextRecoverAt = cooldown.NextRecoverAt
|
||||
dst.BackoffLevel = cooldown.BackoffLevel
|
||||
dst.cooldownSignals = maps.Clone(cooldown.cooldownSignals)
|
||||
dst.cooldownEvaluated = cooldown.Exceeded || cooldown.cooldownEvaluated
|
||||
}
|
||||
|
||||
func rejectedClaudeSignalsForCooldown(provider string, headers http.Header) map[string]string {
|
||||
if !strings.EqualFold(strings.TrimSpace(provider), "claude") {
|
||||
return nil
|
||||
}
|
||||
signals := collectQuotaSignals("claude", headers)
|
||||
if !hasRejectedClaudeUnifiedSignals(signals) {
|
||||
return nil
|
||||
}
|
||||
return signals
|
||||
}
|
||||
|
||||
func sanitizePersistedClaudeCooldownSignals(raw map[string]string) map[string]string {
|
||||
if len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
headers := make(http.Header, len(raw))
|
||||
for key, val := range raw {
|
||||
if strings.HasPrefix(strings.ToLower(strings.TrimSpace(key)), "anthropic-ratelimit-unified-") {
|
||||
headers.Set(key, val)
|
||||
}
|
||||
}
|
||||
return rejectedClaudeSignalsForCooldown("claude", headers)
|
||||
}
|
||||
|
||||
// collectQuotaSignals builds the bounded snapshot for a single response.
|
||||
|
||||
@@ -682,6 +682,11 @@ func (s *authScheduler) mixedUnavailableErrorLocked(providers []string, model st
|
||||
var latestAuthID string
|
||||
var latestAuthErr error
|
||||
|
||||
allClaude := true
|
||||
var earliestClaudeTime time.Time
|
||||
var earliestClaudeID string
|
||||
var earliestClaudeSignals map[string]string
|
||||
|
||||
for _, providerKey := range providers {
|
||||
providerState := s.providers[providerKey]
|
||||
if providerState == nil {
|
||||
@@ -698,6 +703,18 @@ func (s *authScheduler) mixedUnavailableErrorLocked(providers []string, model st
|
||||
if !localEarliest.IsZero() && (earliest.IsZero() || localEarliest.Before(earliest)) {
|
||||
earliest = localEarliest
|
||||
}
|
||||
if localTotal > 0 {
|
||||
localClaude, localClaudeEarliest, localClaudeID, localSignals := shard.claudeCooldownSummaryLocked(providerKey, model, now, predicate)
|
||||
if !localClaude {
|
||||
allClaude = false
|
||||
} else if allClaude && !localClaudeEarliest.IsZero() {
|
||||
if earliestClaudeTime.IsZero() || localClaudeEarliest.Before(earliestClaudeTime) || (localClaudeEarliest.Equal(earliestClaudeTime) && localClaudeID < earliestClaudeID) {
|
||||
earliestClaudeTime = localClaudeEarliest
|
||||
earliestClaudeID = localClaudeID
|
||||
earliestClaudeSignals = localSignals
|
||||
}
|
||||
}
|
||||
}
|
||||
candModelErr, candModelTime, candModelAuthID, candAuthErr, candAuthTime, candAuthAuthID := shard.candidateErrorsLocked(model, predicate)
|
||||
if candModelErr != nil {
|
||||
if latestModelErr == nil || candModelTime.After(latestModelTime) || (candModelTime.Equal(latestModelTime) && candModelAuthID > latestModelAuthID) {
|
||||
@@ -730,7 +747,11 @@ func (s *authScheduler) mixedUnavailableErrorLocked(providers []string, model st
|
||||
if resetIn < 0 {
|
||||
resetIn = 0
|
||||
}
|
||||
return newModelCooldownErrorWithCause(model, "", resetIn, lastCandidateErr)
|
||||
var claudeSignals map[string]string
|
||||
if allClaude {
|
||||
claudeSignals = earliestClaudeSignals
|
||||
}
|
||||
return newModelCooldownErrorForCandidates(model, "", resetIn, lastCandidateErr, allClaude, claudeSignals)
|
||||
}
|
||||
if unauthorizedCount == total {
|
||||
terminalCause := latestAuthErr
|
||||
@@ -1475,7 +1496,8 @@ func (m *modelScheduler) unavailableErrorLocked(provider, model string, predicat
|
||||
if resetIn < 0 {
|
||||
resetIn = 0
|
||||
}
|
||||
return newModelCooldownErrorWithCause(model, providerForError, resetIn, lastCandidateErr)
|
||||
claudeCooldown, _, _, claudeSignals := m.claudeCooldownSummaryLocked(provider, model, now, predicate)
|
||||
return newModelCooldownErrorForCandidates(model, providerForError, resetIn, lastCandidateErr, claudeCooldown, claudeSignals)
|
||||
}
|
||||
if unauthorizedCount == total {
|
||||
terminalCause := authErr
|
||||
@@ -1591,6 +1613,57 @@ func (m *modelScheduler) availabilitySummaryLocked(predicate func(*scheduledAuth
|
||||
return total, cooldownCount, unauthorizedCount, earliest
|
||||
}
|
||||
|
||||
func (m *modelScheduler) claudeCooldownSummaryLocked(fallbackProvider, model string, now time.Time, predicate func(*scheduledAuth) bool) (bool, time.Time, string, map[string]string) {
|
||||
if m == nil {
|
||||
return false, time.Time{}, "", nil
|
||||
}
|
||||
matched := 0
|
||||
var earliest time.Time
|
||||
var earliestID string
|
||||
var earliestAuth *Auth
|
||||
var earliestSignals map[string]string
|
||||
for _, entry := range m.entries {
|
||||
if predicate != nil && !predicate(entry) {
|
||||
continue
|
||||
}
|
||||
matched++
|
||||
if entry == nil || entry.auth == nil {
|
||||
return false, time.Time{}, "", nil
|
||||
}
|
||||
entryProvider := ""
|
||||
if entry.meta != nil && entry.meta.providerKey != "" {
|
||||
entryProvider = entry.meta.providerKey
|
||||
} else if resolved := executorKeyFromAuth(entry.auth); resolved != "" {
|
||||
entryProvider = resolved
|
||||
} else {
|
||||
entryProvider = strings.ToLower(strings.TrimSpace(fallbackProvider))
|
||||
}
|
||||
if entryProvider != "claude" {
|
||||
return false, time.Time{}, "", nil
|
||||
}
|
||||
checkModel := model
|
||||
if existingModelState(entry.auth, checkModel) == nil && m.modelKey != "" {
|
||||
checkModel = m.modelKey
|
||||
}
|
||||
okClaude, signals := claudeUnifiedCooldownForAuth(entry.auth, checkModel, now)
|
||||
if !okClaude {
|
||||
return false, time.Time{}, "", nil
|
||||
}
|
||||
if entry.state == scheduledStateCooldown && !entry.nextRetryAt.IsZero() {
|
||||
if earliestAuth == nil || entry.nextRetryAt.Before(earliest) || (entry.nextRetryAt.Equal(earliest) && entry.auth.ID < earliestID) {
|
||||
earliest = entry.nextRetryAt
|
||||
earliestID = entry.auth.ID
|
||||
earliestAuth = entry.auth
|
||||
earliestSignals = signals
|
||||
}
|
||||
}
|
||||
}
|
||||
if matched == 0 || earliestAuth == nil {
|
||||
return false, time.Time{}, "", nil
|
||||
}
|
||||
return true, earliest, earliestID, earliestSignals
|
||||
}
|
||||
|
||||
// rebuildIndexesLocked reconstructs ready and blocked views from the current entry map.
|
||||
func (m *modelScheduler) rebuildIndexesLocked() {
|
||||
cursorStates := make(map[int]readyBucketCursorState, len(m.readyByPriority))
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"maps"
|
||||
"math"
|
||||
"net/http"
|
||||
"regexp"
|
||||
@@ -83,10 +84,12 @@ const (
|
||||
)
|
||||
|
||||
type modelCooldownError struct {
|
||||
model string
|
||||
resetIn time.Duration
|
||||
provider string
|
||||
cause error
|
||||
model string
|
||||
resetIn time.Duration
|
||||
provider string
|
||||
cause error
|
||||
claudeCooldown bool
|
||||
claudeSignals map[string]string
|
||||
}
|
||||
|
||||
// NewModelCooldownError creates an error representing model-level cooldown.
|
||||
@@ -99,17 +102,27 @@ func newModelCooldownError(model, provider string, resetIn time.Duration) *model
|
||||
}
|
||||
|
||||
func newModelCooldownErrorWithCause(model, provider string, resetIn time.Duration, cause error) *modelCooldownError {
|
||||
return newModelCooldownErrorForCandidates(model, provider, resetIn, cause, isClaudeCooldownProvider(provider), nil)
|
||||
}
|
||||
|
||||
func newModelCooldownErrorForCandidates(model, provider string, resetIn time.Duration, cause error, claudeCooldown bool, claudeSignals map[string]string) *modelCooldownError {
|
||||
if resetIn < 0 {
|
||||
resetIn = 0
|
||||
}
|
||||
return &modelCooldownError{
|
||||
model: model,
|
||||
provider: provider,
|
||||
resetIn: resetIn,
|
||||
cause: cause,
|
||||
model: model,
|
||||
provider: provider,
|
||||
resetIn: resetIn,
|
||||
cause: cause,
|
||||
claudeCooldown: claudeCooldown,
|
||||
claudeSignals: maps.Clone(claudeSignals),
|
||||
}
|
||||
}
|
||||
|
||||
func isClaudeCooldownProvider(provider string) bool {
|
||||
return strings.EqualFold(strings.TrimSpace(provider), "claude")
|
||||
}
|
||||
|
||||
func (e *modelCooldownError) IsModelCooldown() bool {
|
||||
return true
|
||||
}
|
||||
@@ -352,17 +365,419 @@ func (e *modelCooldownError) StatusCode() int {
|
||||
return http.StatusTooManyRequests
|
||||
}
|
||||
|
||||
func (e *modelCooldownError) Headers() http.Header {
|
||||
func (e *modelCooldownError) resetSeconds() int64 {
|
||||
if e == nil || e.resetIn <= 0 {
|
||||
return 0
|
||||
}
|
||||
seconds := int64(e.resetIn / time.Second)
|
||||
if e.resetIn%time.Second > 0 {
|
||||
seconds++
|
||||
}
|
||||
return seconds
|
||||
}
|
||||
|
||||
// SafeResponseHeaders exposes only the locally trusted cooldown headers.
|
||||
func (e *modelCooldownError) SafeResponseHeaders() http.Header {
|
||||
headers := make(http.Header)
|
||||
headers.Set("Content-Type", "application/json")
|
||||
resetSeconds := int(math.Ceil(e.resetIn.Seconds()))
|
||||
if resetSeconds < 0 {
|
||||
resetSeconds = 0
|
||||
}
|
||||
headers.Set("Retry-After", strconv.Itoa(resetSeconds))
|
||||
headers.Set("Retry-After", strconv.FormatInt(e.resetSeconds(), 10))
|
||||
return headers
|
||||
}
|
||||
|
||||
func (e *modelCooldownError) Headers() http.Header {
|
||||
headers := e.SafeResponseHeaders()
|
||||
if e != nil && e.claudeCooldown && e.resetIn > 0 {
|
||||
e.appendClaudeUnifiedHeaders(headers)
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
func (e *modelCooldownError) appendClaudeUnifiedHeaders(headers http.Header) {
|
||||
resetSeconds := e.resetSeconds()
|
||||
if resetSeconds <= 0 {
|
||||
return
|
||||
}
|
||||
resetAt := time.Now().Unix() + resetSeconds
|
||||
resetStr := strconv.FormatInt(resetAt, 10)
|
||||
|
||||
if hasRejectedClaudeUnifiedSignals(e.claudeSignals) {
|
||||
for key, value := range e.claudeSignals {
|
||||
if strings.HasPrefix(strings.ToLower(strings.TrimSpace(key)), "anthropic-ratelimit-unified-") {
|
||||
headers.Set(key, value)
|
||||
}
|
||||
}
|
||||
headers.Set("Anthropic-Ratelimit-Unified-Status", "rejected")
|
||||
headers.Set("Anthropic-Ratelimit-Unified-Reset", resetStr)
|
||||
|
||||
rejWindows := rejectedClaudeUnifiedWindows(headers)
|
||||
claim := strings.ToLower(strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-Representative-Claim")))
|
||||
if len(rejWindows) == 0 && claim == "" && isClaudeOverageOnlyHeaders(headers) {
|
||||
claim = "overage"
|
||||
headers.Set("Anthropic-Ratelimit-Unified-Representative-Claim", claim)
|
||||
}
|
||||
if len(rejWindows) == 0 && claim != "overage" {
|
||||
fallbackWindow := fallbackClaudeWindowForClaim(headers, claim, e.resetIn)
|
||||
if fallbackWindow != "" {
|
||||
statusHeader := claudeUnifiedWindowHeader(fallbackWindow, "Status")
|
||||
if strings.TrimSpace(headers.Get(statusHeader)) == "" {
|
||||
headers.Set(statusHeader, "rejected")
|
||||
rejWindows = append(rejWindows, fallbackWindow)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(rejWindows) == 1 {
|
||||
headers.Set(claudeUnifiedWindowHeader(rejWindows[0], "Reset"), resetStr)
|
||||
} else if len(rejWindows) > 1 {
|
||||
latestHeader := ""
|
||||
var latestEpoch int64
|
||||
for _, window := range rejWindows {
|
||||
resetHeader := claudeUnifiedWindowHeader(window, "Reset")
|
||||
raw := strings.TrimSpace(headers.Get(resetHeader))
|
||||
if raw == "" {
|
||||
headers.Set(resetHeader, resetStr)
|
||||
continue
|
||||
}
|
||||
if parsed, errParse := strconv.ParseInt(raw, 10, 64); errParse == nil && (latestHeader == "" || parsed >= latestEpoch) {
|
||||
latestHeader = resetHeader
|
||||
latestEpoch = parsed
|
||||
}
|
||||
}
|
||||
if latestHeader != "" {
|
||||
headers.Set(latestHeader, resetStr)
|
||||
}
|
||||
}
|
||||
|
||||
if claim == "" {
|
||||
if inferred := inferClaudeRepresentativeClaim(headers, rejWindows, e.resetIn); inferred != "" {
|
||||
headers.Set("Anthropic-Ratelimit-Unified-Representative-Claim", inferred)
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-Overage-Status")) == "" {
|
||||
headers.Set("Anthropic-Ratelimit-Unified-Overage-Status", "rejected")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
claim := "five_hour"
|
||||
if e.resetIn > 5*time.Hour {
|
||||
claim = "seven_day"
|
||||
}
|
||||
headers.Set("Anthropic-Ratelimit-Unified-Status", "rejected")
|
||||
headers.Set("Anthropic-Ratelimit-Unified-Reset", resetStr)
|
||||
headers.Set("Anthropic-Ratelimit-Unified-Representative-Claim", claim)
|
||||
headers.Set("Anthropic-Ratelimit-Unified-Overage-Status", "rejected")
|
||||
if claim == "five_hour" {
|
||||
headers.Set("Anthropic-Ratelimit-Unified-5h-Status", "rejected")
|
||||
headers.Set("Anthropic-Ratelimit-Unified-5h-Reset", resetStr)
|
||||
} else {
|
||||
headers.Set("Anthropic-Ratelimit-Unified-7d-Status", "rejected")
|
||||
headers.Set("Anthropic-Ratelimit-Unified-7d-Reset", resetStr)
|
||||
}
|
||||
}
|
||||
|
||||
func claudeUnifiedWindowName(headerKey string) (string, bool) {
|
||||
lower := strings.ToLower(strings.TrimSpace(headerKey))
|
||||
rest, hasPrefix := strings.CutPrefix(lower, "anthropic-ratelimit-unified-")
|
||||
if !hasPrefix {
|
||||
return "", false
|
||||
}
|
||||
window, hasSuffix := strings.CutSuffix(rest, "-status")
|
||||
if !hasSuffix || window == "" || window == "overage" {
|
||||
return "", false
|
||||
}
|
||||
return window, true
|
||||
}
|
||||
|
||||
func claudeUnifiedWindowHeader(window, field string) string {
|
||||
return http.CanonicalHeaderKey("Anthropic-Ratelimit-Unified-" + window + "-" + field)
|
||||
}
|
||||
|
||||
func rejectedClaudeUnifiedWindows(headers http.Header) []string {
|
||||
var windows []string
|
||||
for key := range headers {
|
||||
window, ok := claudeUnifiedWindowName(key)
|
||||
if ok && strings.EqualFold(strings.TrimSpace(headers.Get(key)), "rejected") {
|
||||
windows = append(windows, window)
|
||||
}
|
||||
}
|
||||
sort.Strings(windows)
|
||||
return windows
|
||||
}
|
||||
|
||||
func isAllowedClaudeWindowStatus(status string) bool {
|
||||
lower := strings.ToLower(strings.TrimSpace(status))
|
||||
return lower == "allowed" || lower == "allowed_warning"
|
||||
}
|
||||
|
||||
func isClaudeUtilizationBelowCap(raw string) bool {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return false
|
||||
}
|
||||
u, errParse := strconv.ParseFloat(raw, 64)
|
||||
if errParse != nil || math.IsNaN(u) || math.IsInf(u, 0) {
|
||||
return false
|
||||
}
|
||||
return u >= 0 && u < 1.0
|
||||
}
|
||||
|
||||
func isClaudeOverageOnlyHeaders(headers http.Header) bool {
|
||||
overageRejected := strings.EqualFold(strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-Overage-Status")), "rejected") ||
|
||||
strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-Overage-Disabled-Reason")) != ""
|
||||
if !overageRejected {
|
||||
return false
|
||||
}
|
||||
status5h := strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-5h-Status"))
|
||||
status7d := strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-7d-Status"))
|
||||
allowed5h := isAllowedClaudeWindowStatus(status5h)
|
||||
allowed7d := isAllowedClaudeWindowStatus(status7d)
|
||||
if allowed5h && allowed7d {
|
||||
return true
|
||||
}
|
||||
if allowed7d && status5h == "" {
|
||||
util5h := strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-5h-Utilization"))
|
||||
if util5h == "" || isClaudeUtilizationBelowCap(util5h) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if allowed5h && status7d == "" && isClaudeUtilizationBelowCap(headers.Get("Anthropic-Ratelimit-Unified-7d-Utilization")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func fallbackClaudeWindowForClaim(headers http.Header, claim string, resetIn time.Duration) string {
|
||||
switch {
|
||||
case claim == "seven_day_overage_included":
|
||||
return "7d_oi"
|
||||
case strings.HasPrefix(claim, "seven_day_"):
|
||||
if suffix := strings.TrimPrefix(claim, "seven_day_"); suffix != "" {
|
||||
return "7d_" + suffix
|
||||
}
|
||||
case claim == "seven_day":
|
||||
return "7d"
|
||||
case claim == "five_hour":
|
||||
return "5h"
|
||||
case claim == "":
|
||||
allowed5h := isAllowedClaudeWindowStatus(headers.Get("Anthropic-Ratelimit-Unified-5h-Status"))
|
||||
allowed7d := isAllowedClaudeWindowStatus(headers.Get("Anthropic-Ratelimit-Unified-7d-Status"))
|
||||
switch {
|
||||
case allowed5h && allowed7d:
|
||||
return ""
|
||||
case allowed5h:
|
||||
return "7d"
|
||||
case allowed7d:
|
||||
return "5h"
|
||||
case resetIn > 5*time.Hour:
|
||||
return "7d"
|
||||
default:
|
||||
return "5h"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func inferClaudeRepresentativeClaim(headers http.Header, windows []string, resetIn time.Duration) string {
|
||||
has5h := false
|
||||
has7d := false
|
||||
has7dOI := false
|
||||
modelFamily7d := ""
|
||||
otherWindow := ""
|
||||
for _, window := range windows {
|
||||
switch {
|
||||
case window == "7d_oi":
|
||||
has7dOI = true
|
||||
case strings.HasPrefix(window, "7d_"):
|
||||
if modelFamily7d == "" {
|
||||
modelFamily7d = "seven_day_" + strings.TrimPrefix(window, "7d_")
|
||||
}
|
||||
case window == "7d":
|
||||
has7d = true
|
||||
case window == "5h":
|
||||
has5h = true
|
||||
default:
|
||||
if otherWindow == "" {
|
||||
otherWindow = window
|
||||
}
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case has7dOI:
|
||||
return "seven_day_overage_included"
|
||||
case modelFamily7d != "":
|
||||
return modelFamily7d
|
||||
case has7d:
|
||||
return "seven_day"
|
||||
case has5h:
|
||||
return "five_hour"
|
||||
case otherWindow != "":
|
||||
return otherWindow
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-Overage-Status")), "rejected") ||
|
||||
strings.TrimSpace(headers.Get("Anthropic-Ratelimit-Unified-Overage-Disabled-Reason")) != "" {
|
||||
return "overage"
|
||||
}
|
||||
allowed5h := isAllowedClaudeWindowStatus(headers.Get("Anthropic-Ratelimit-Unified-5h-Status"))
|
||||
allowed7d := isAllowedClaudeWindowStatus(headers.Get("Anthropic-Ratelimit-Unified-7d-Status"))
|
||||
switch {
|
||||
case allowed5h && allowed7d:
|
||||
return ""
|
||||
case allowed5h:
|
||||
return "seven_day"
|
||||
case allowed7d:
|
||||
return "five_hour"
|
||||
case resetIn > 5*time.Hour:
|
||||
return "seven_day"
|
||||
default:
|
||||
return "five_hour"
|
||||
}
|
||||
}
|
||||
|
||||
func hasRejectedClaudeUnifiedSignals(signals map[string]string) bool {
|
||||
if len(signals) == 0 {
|
||||
return false
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(signals["Anthropic-Ratelimit-Unified-Status"]), "rejected") {
|
||||
return true
|
||||
}
|
||||
for key, val := range signals {
|
||||
if _, ok := claudeUnifiedWindowName(key); ok && strings.EqualFold(strings.TrimSpace(val), "rejected") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func claudeSignalsFromEvaluatedQuota(q QuotaState) (bool, map[string]string) {
|
||||
if hasRejectedClaudeUnifiedSignals(q.cooldownSignals) {
|
||||
return true, maps.Clone(q.cooldownSignals)
|
||||
}
|
||||
if q.cooldownEvaluated {
|
||||
return false, nil
|
||||
}
|
||||
if hasRejectedClaudeUnifiedSignals(q.Signals) {
|
||||
return true, maps.Clone(q.Signals)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func claudeUnifiedCooldownForAuth(auth *Auth, model string, now time.Time) (bool, map[string]string) {
|
||||
if auth == nil || executorKeyFromAuth(auth) != "claude" {
|
||||
return false, nil
|
||||
}
|
||||
if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now) {
|
||||
state := existingModelState(auth, model)
|
||||
if state != nil && state.NextRetryAfter.After(auth.Quota.NextRecoverAt) {
|
||||
if shadow := state.credentialQuotaShadow; shadow != nil {
|
||||
if !shadow.Quota.Exceeded || !shadow.Quota.NextRecoverAt.After(now) || shadow.NextRetryAfter.After(shadow.Quota.NextRecoverAt) {
|
||||
return false, nil
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(shadow.Quota.Reason), "cloudflare challenge") {
|
||||
return false, nil
|
||||
}
|
||||
return claudeSignalsFromEvaluatedQuota(shadow.Quota)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
if ok, signals := claudeSignalsFromEvaluatedQuota(auth.Quota); ok {
|
||||
return true, signals
|
||||
}
|
||||
if state != nil && !state.Quota.NextRecoverAt.Before(auth.Quota.NextRecoverAt) {
|
||||
if ok, signals := claudeSignalsFromEvaluatedQuota(state.Quota); ok {
|
||||
return true, signals
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
if len(auth.ModelStates) > 0 {
|
||||
if model == "" {
|
||||
if !auth.Unavailable {
|
||||
return false, nil
|
||||
}
|
||||
keys := make([]string, 0, len(auth.ModelStates))
|
||||
for key := range auth.ModelStates {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var earliestRecover time.Time
|
||||
var earliestSignals map[string]string
|
||||
found := false
|
||||
for _, key := range keys {
|
||||
st := auth.ModelStates[key]
|
||||
if st != nil && (st.credentialQuotaPropagated || st.Quota.Reason == "credential_quota") {
|
||||
st = st.credentialQuotaShadow
|
||||
}
|
||||
if st == nil || !st.Quota.Exceeded || !st.Quota.NextRecoverAt.After(now) || st.NextRetryAfter.After(st.Quota.NextRecoverAt) {
|
||||
return false, nil
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(st.Quota.Reason), "cloudflare challenge") {
|
||||
return false, nil
|
||||
}
|
||||
okSt, sig := claudeSignalsFromEvaluatedQuota(st.Quota)
|
||||
if !okSt {
|
||||
return false, nil
|
||||
}
|
||||
if !found || st.Quota.NextRecoverAt.Before(earliestRecover) {
|
||||
found = true
|
||||
earliestRecover = st.Quota.NextRecoverAt
|
||||
earliestSignals = sig
|
||||
}
|
||||
}
|
||||
return found, earliestSignals
|
||||
}
|
||||
state := existingModelState(auth, model)
|
||||
if state != nil && (state.credentialQuotaPropagated || state.Quota.Reason == "credential_quota") {
|
||||
state = state.credentialQuotaShadow
|
||||
}
|
||||
if state == nil || !state.Quota.Exceeded || !state.Quota.NextRecoverAt.After(now) || state.NextRetryAfter.After(state.Quota.NextRecoverAt) {
|
||||
return false, nil
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(state.Quota.Reason), "cloudflare challenge") {
|
||||
return false, nil
|
||||
}
|
||||
return claudeSignalsFromEvaluatedQuota(state.Quota)
|
||||
}
|
||||
if !auth.Quota.Exceeded || !auth.Quota.NextRecoverAt.After(now) || auth.NextRetryAfter.After(auth.Quota.NextRecoverAt) {
|
||||
return false, nil
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(auth.Quota.Reason), "cloudflare challenge") {
|
||||
return false, nil
|
||||
}
|
||||
return claudeSignalsFromEvaluatedQuota(auth.Quota)
|
||||
}
|
||||
|
||||
func claudeCooldownMetadataForAuths(ctx context.Context, auths []*Auth, modelForAuth func(*Auth) string, now time.Time, clientHeaders ...http.Header) (bool, map[string]string) {
|
||||
if len(auths) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
var earliestAuth *Auth
|
||||
var earliestSignals map[string]string
|
||||
var earliestNext time.Time
|
||||
for _, candidate := range auths {
|
||||
checkModel := ""
|
||||
if modelForAuth != nil {
|
||||
checkModel = modelForAuth(candidate)
|
||||
}
|
||||
okClaude, signals := claudeUnifiedCooldownForAuth(candidate, checkModel, now)
|
||||
if !okClaude {
|
||||
return false, nil
|
||||
}
|
||||
blocked, reason, next := isAuthBlockedForModelForRequest(ctx, candidate, checkModel, now, clientHeaders...)
|
||||
if blocked && reason == blockReasonCooldown && next.After(now) {
|
||||
if earliestAuth == nil || next.Before(earliestNext) || (next.Equal(earliestNext) && candidate.ID < earliestAuth.ID) {
|
||||
earliestAuth = candidate
|
||||
earliestSignals = signals
|
||||
earliestNext = next
|
||||
}
|
||||
}
|
||||
}
|
||||
if earliestAuth == nil {
|
||||
return false, nil
|
||||
}
|
||||
return true, earliestSignals
|
||||
}
|
||||
|
||||
func authPriority(auth *Auth) int {
|
||||
if auth == nil || auth.Attributes == nil {
|
||||
return 0
|
||||
@@ -558,7 +973,10 @@ func getAvailableAuthsWithPriorityModeForRequest(ctx context.Context, auths []*A
|
||||
if resetIn < 0 {
|
||||
resetIn = 0
|
||||
}
|
||||
return nil, newModelCooldownError(model, providerForError, resetIn)
|
||||
claudeCooldown, claudeSignals := claudeCooldownMetadataForAuths(ctx, auths, func(*Auth) string {
|
||||
return model
|
||||
}, now, clientHeaders...)
|
||||
return nil, newModelCooldownErrorForCandidates(model, providerForError, resetIn, nil, claudeCooldown, claudeSignals)
|
||||
}
|
||||
return nil, newAuthUnavailableError(earliest, now)
|
||||
}
|
||||
@@ -819,7 +1237,23 @@ func isAuthBlockedForModelForRequest(ctx context.Context, auth *Auth, model stri
|
||||
return true, blockReasonOther, time.Time{}
|
||||
}
|
||||
if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now) {
|
||||
return true, blockReasonCooldown, auth.Quota.NextRecoverAt
|
||||
credNext := auth.Quota.NextRecoverAt
|
||||
if model != "" && len(auth.ModelStates) > 0 {
|
||||
modelKey := canonicalModelLookupKey(model)
|
||||
for stateModel, state := range auth.ModelStates {
|
||||
if state == nil || canonicalModelLookupKey(stateModel) != modelKey {
|
||||
continue
|
||||
}
|
||||
if state.Status == StatusDisabled {
|
||||
return true, blockReasonDisabled, time.Time{}
|
||||
}
|
||||
stateBlocked, _, next := availabilityBlock(state.Unavailable, state.Quota.Exceeded, state.NextRetryAfter, state.Quota.NextRecoverAt, now)
|
||||
if stateBlocked && next.After(credNext) {
|
||||
credNext = next
|
||||
}
|
||||
}
|
||||
}
|
||||
return true, blockReasonCooldown, credNext
|
||||
}
|
||||
unavailable := codexRefreshUnavailableForRequest(ctx, auth, now, clientHeaders...)
|
||||
if model != "" {
|
||||
|
||||
@@ -355,6 +355,10 @@ type QuotaState struct {
|
||||
// Cooldown transitions must use applyCooldownFields so they cannot replace
|
||||
// this snapshot.
|
||||
Signals map[string]string `json:"signals,omitempty"`
|
||||
// cooldownSignals retains rejected quota headers captured when the active
|
||||
// cooldown was recorded so out-of-order completions cannot overwrite them.
|
||||
cooldownSignals map[string]string
|
||||
cooldownEvaluated bool
|
||||
}
|
||||
|
||||
// Clone returns an independent copy of the quota state.
|
||||
@@ -366,6 +370,12 @@ func (q QuotaState) Clone() QuotaState {
|
||||
copyQuota.Signals[key] = value
|
||||
}
|
||||
}
|
||||
if len(q.cooldownSignals) > 0 {
|
||||
copyQuota.cooldownSignals = make(map[string]string, len(q.cooldownSignals))
|
||||
for key, value := range q.cooldownSignals {
|
||||
copyQuota.cooldownSignals[key] = value
|
||||
}
|
||||
}
|
||||
return copyQuota
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user