diff --git a/sdk/cliproxy/auth/conductor_cooldown.go b/sdk/cliproxy/auth/conductor_cooldown.go index 272c6c2f5b..e41f32b352 100644 --- a/sdk/cliproxy/auth/conductor_cooldown.go +++ b/sdk/cliproxy/auth/conductor_cooldown.go @@ -867,7 +867,14 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) { } case 429: var next time.Time + var credentialNext time.Time backoffLevel := state.Quota.BackoffLevel + if result.CredentialScope { + backoffLevel = 0 + if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" { + backoffLevel = auth.Quota.BackoffLevel + } + } if !disableCooling { if result.RetryAfter != nil { cooldown := *result.RetryAfter @@ -876,8 +883,18 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) { } next = now.Add(cooldown).Round(0) } else { - next, backoffLevel = quotaCooldownAfterFailure(state.Quota, now) + quotaForFailure := state.Quota + if result.CredentialScope { + if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" { + quotaForFailure = auth.Quota + } else { + quotaForFailure.NextRecoverAt = time.Time{} + quotaForFailure.BackoffLevel = 0 + } + } + next, backoffLevel = quotaCooldownAfterFailure(quotaForFailure, now) } + credentialNext = next if state.Quota.Exceeded && state.Quota.NextRecoverAt.After(next) { next = state.Quota.NextRecoverAt } @@ -894,7 +911,7 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) { if otherState != nil && otherState != state { otherState.Unavailable = true otherState.Status = StatusError - otherQuotaNext := next + otherQuotaNext := credentialNext if otherState.Quota.Exceeded && otherState.Quota.NextRecoverAt.After(otherQuotaNext) { otherQuotaNext = otherState.Quota.NextRecoverAt } @@ -914,13 +931,15 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) { } } auth.Unavailable = true - auth.Quota.Exceeded = true - auth.Quota.Reason = "credential_quota" - authNext := next - if auth.Quota.NextRecoverAt.After(authNext) { + authNext := credentialNext + if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && + auth.Quota.NextRecoverAt.After(authNext) { authNext = auth.Quota.NextRecoverAt } + auth.Quota.Exceeded = true + auth.Quota.Reason = "credential_quota" auth.Quota.NextRecoverAt = authNext + auth.Quota.BackoffLevel = backoffLevel auth.NextRetryAfter = authNext } case 408, 500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526: diff --git a/sdk/cliproxy/auth/conductor_cooldown_monotonic_test.go b/sdk/cliproxy/auth/conductor_cooldown_monotonic_test.go index 6f3c70b7ee..b6e7fd2357 100644 --- a/sdk/cliproxy/auth/conductor_cooldown_monotonic_test.go +++ b/sdk/cliproxy/auth/conductor_cooldown_monotonic_test.go @@ -283,3 +283,172 @@ func TestManager_ApplyAuthFailureState_PreservesLongerCredentialDeadline(t *test }) } } + +func TestManager_MarkResult_CredentialScopeDoesNotInheritModelQuotaDeadline(t *testing.T) { + withQuotaCooldownEnabled(t) + + testCases := []struct { + name string + credentialModel string + }{ + {name: "cross_model", credentialModel: "claude-sonnet-5"}, + {name: "same_model", credentialModel: "claude-fable-5-1"}, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + m, auth := newCooldownMonotonicManager(t, "claude-fable-5-1", "claude-sonnet-5", "claude-opus-5-5", "claude-sonnet-4") + ctx := context.Background() + for _, model := range []string{"claude-fable-5-1", "claude-sonnet-5", "claude-opus-5-5"} { + m.MarkResult(ctx, Result{AuthID: auth.ID, Provider: auth.Provider, Model: model, Success: true}) + } + + before := time.Now() + modelOnlyRetry := 8 * 24 * time.Hour + m.MarkResult(ctx, Result{ + AuthID: auth.ID, Provider: auth.Provider, Model: "claude-fable-5-1", + RetryAfter: &modelOnlyRetry, + Error: &Error{HTTPStatus: http.StatusTooManyRequests, Message: "Usage credits are required for this model."}, + }) + + modelOnlyAuth, _ := m.GetByID(auth.ID) + if modelOnlyAuth.Quota.Reason != "quota" || !modelOnlyAuth.Quota.NextRecoverAt.After(before.Add(7*24*time.Hour)) { + t.Fatalf("precondition failed: model quota deadline was not aggregated: quota=%+v", modelOnlyAuth.Quota) + } + if blocked, _, _ := isAuthBlockedForModel(modelOnlyAuth, "claude-opus-5-5", time.Now()); blocked { + t.Fatal("model-only cooldown should not block an unrelated model") + } + + credentialRetry := 3 * time.Hour + m.MarkResult(ctx, Result{ + AuthID: auth.ID, Provider: auth.Provider, Model: testCase.credentialModel, + RetryAfter: &credentialRetry, CredentialScope: true, + Error: &Error{HTTPStatus: http.StatusTooManyRequests, Message: "shared window rejected"}, + }) + + updated, _ := m.GetByID(auth.ID) + minCredentialDeadline := before.Add(credentialRetry - time.Hour) + maxCredentialDeadline := before.Add(credentialRetry + time.Hour) + if updated.Quota.Reason != "credential_quota" || + updated.Quota.NextRecoverAt.Before(minCredentialDeadline) || + updated.Quota.NextRecoverAt.After(maxCredentialDeadline) { + t.Fatalf("credential cooldown inherited a model-only deadline: reason=%q deadline=%v (in %v)", + updated.Quota.Reason, updated.Quota.NextRecoverAt, updated.Quota.NextRecoverAt.Sub(before)) + } + + opusBlocked, _, opusNext := isAuthBlockedForModel(updated, "claude-opus-5-5", before.Add(credentialRetry+time.Minute)) + if opusBlocked { + t.Fatalf("opus remained blocked beyond credential cooldown: next=%v (in %v)", opusNext, opusNext.Sub(before)) + } + fableBlocked, _, fableNext := isAuthBlockedForModel(updated, "claude-fable-5-1", before.Add(credentialRetry+time.Minute)) + if !fableBlocked || !fableNext.After(before.Add(7*24*time.Hour)) { + t.Fatalf("model-only cooldown was not retained for Fable: blocked=%v next=%v", fableBlocked, fableNext) + } + + credentialDeadline := updated.Quota.NextRecoverAt + shorterCredentialRetry := 30 * time.Minute + m.MarkResult(ctx, Result{ + AuthID: auth.ID, Provider: auth.Provider, Model: "claude-sonnet-4", + RetryAfter: &shorterCredentialRetry, CredentialScope: true, + Error: &Error{HTTPStatus: http.StatusTooManyRequests, Message: "shorter shared window rejection"}, + }) + updated, _ = m.GetByID(auth.ID) + if !updated.Quota.NextRecoverAt.Equal(credentialDeadline) { + t.Fatalf("later credential-scoped cooldown shortened the active credential deadline: got=%v want=%v", + updated.Quota.NextRecoverAt, credentialDeadline) + } + }) + } +} + +func TestManager_MarkResult_CredentialScopeBackoffPersistsAcrossWindows(t *testing.T) { + withQuotaCooldownEnabled(t) + + m, auth := newCooldownMonotonicManager(t, "model-a", "model-b") + ctx := context.Background() + m.MarkResult(ctx, Result{ + AuthID: auth.ID, Provider: auth.Provider, Model: "model-a", CredentialScope: true, + Error: &Error{HTTPStatus: http.StatusTooManyRequests, Message: "credential quota exceeded"}, + }) + + updated, _ := m.GetByID(auth.ID) + if updated.Quota.BackoffLevel != 1 { + t.Fatalf("first credential quota failure did not retain backoff level 1: quota=%+v", updated.Quota) + } + + expired := time.Now().Add(-time.Minute) + m.mu.Lock() + storedAuth := m.auths[auth.ID] + storedAuth.Quota.NextRecoverAt = expired + storedAuth.NextRetryAfter = expired + for _, state := range storedAuth.ModelStates { + if state != nil { + state.Quota.NextRecoverAt = expired + state.NextRetryAfter = expired + } + } + m.mu.Unlock() + + secondFailureAt := time.Now() + m.MarkResult(ctx, Result{ + AuthID: auth.ID, Provider: auth.Provider, Model: "model-b", CredentialScope: true, + Error: &Error{HTTPStatus: http.StatusTooManyRequests, Message: "credential quota exceeded again"}, + }) + + updated, _ = m.GetByID(auth.ID) + if updated.Quota.BackoffLevel != 2 { + t.Fatalf("second credential quota failure did not advance backoff to level 2: quota=%+v", updated.Quota) + } + if updated.Quota.NextRecoverAt.Before(secondFailureAt.Add(2 * quotaBackoffBase)) { + t.Fatalf("second credential quota failure did not use the next backoff window: deadline=%v", updated.Quota.NextRecoverAt) + } +} + +func TestManager_MarkResult_CredentialScopeDoesNotInheritModelBackoffLevel(t *testing.T) { + withQuotaCooldownEnabled(t) + + retryAfter := 10 * time.Second + tests := []struct { + name string + retryAfter *time.Duration + wantBackoffLevel int + wantCooldownDuration time.Duration + }{ + {name: "without_retry_hint", wantBackoffLevel: 1, wantCooldownDuration: quotaBackoffBase}, + {name: "with_retry_hint", retryAfter: &retryAfter, wantCooldownDuration: retryAfter}, + } + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + m, auth := newCooldownMonotonicManager(t, "model-a", "model-b") + ctx := context.Background() + m.MarkResult(ctx, Result{ + AuthID: auth.ID, Provider: auth.Provider, Model: "model-a", + Error: &Error{HTTPStatus: http.StatusTooManyRequests, Message: "model-only quota exceeded"}, + }) + + modelOnlyAuth, _ := m.GetByID(auth.ID) + modelState := existingModelState(modelOnlyAuth, canonicalModelKey("model-a")) + if modelOnlyAuth.Quota.Reason != "quota" || modelState == nil || modelState.Quota.BackoffLevel != 1 { + t.Fatalf("precondition failed: model-only quota/backoff missing: auth=%+v state=%+v", modelOnlyAuth.Quota, modelState) + } + + secondFailureStarted := time.Now() + m.MarkResult(ctx, Result{ + AuthID: auth.ID, Provider: auth.Provider, Model: "model-a", RetryAfter: testCase.retryAfter, + CredentialScope: true, + Error: &Error{HTTPStatus: http.StatusTooManyRequests, Message: "credential quota exceeded"}, + }) + secondFailureCompleted := time.Now() + + updated, _ := m.GetByID(auth.ID) + if updated.Quota.BackoffLevel != testCase.wantBackoffLevel { + t.Fatalf("credential cooldown inherited model backoff: got level %d, want %d", updated.Quota.BackoffLevel, testCase.wantBackoffLevel) + } + minDeadline := secondFailureStarted.Add(testCase.wantCooldownDuration) + maxDeadline := secondFailureCompleted.Add(testCase.wantCooldownDuration) + if updated.Quota.NextRecoverAt.Before(minDeadline) || updated.Quota.NextRecoverAt.After(maxDeadline) { + t.Fatalf("credential cooldown deadline=%v, want between %v and %v", + updated.Quota.NextRecoverAt, minDeadline, maxDeadline) + } + }) + } +}