fix(auth): isolate credential-scoped quota cooldown from model quota state

- Track credential cooldown deadlines independently from individual model quota recovery deadlines - Use dedicated credential backoff levels and state instead of inheriting model-specific backoff during 429 handling - Ensure auth-level quota recovery timestamps only retain previous deadlines when the existing reason is `credential_quota` Closes: #6083
2026-10-25 00:56:52 +08:00 · 2026-09-24 00:10:54 +08:00
parent 5af6cd7582
commit aafa4e9549
2 changed files with 194 additions and 6 deletions
--- a/sdk/cliproxy/auth/conductor_cooldown.go
+++ b/sdk/cliproxy/auth/conductor_cooldown.go
@@ -867,7 +867,14 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
 							}
 						case 429:
 							var next time.Time
+							var credentialNext time.Time
 							backoffLevel := state.Quota.BackoffLevel
+							if result.CredentialScope {
+								backoffLevel = 0
+								if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" {
+									backoffLevel = auth.Quota.BackoffLevel
+								}
+							}
 							if !disableCooling {
 								if result.RetryAfter != nil {
 									cooldown := *result.RetryAfter
@@ -876,8 +883,18 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
 									}
 									next = now.Add(cooldown).Round(0)
 								} else {
-									next, backoffLevel = quotaCooldownAfterFailure(state.Quota, now)
+									quotaForFailure := state.Quota
+									if result.CredentialScope {
+										if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" {
+											quotaForFailure = auth.Quota
+										} else {
+											quotaForFailure.NextRecoverAt = time.Time{}
+											quotaForFailure.BackoffLevel = 0
+										}
+									}
+									next, backoffLevel = quotaCooldownAfterFailure(quotaForFailure, now)
 								}
+								credentialNext = next
 								if state.Quota.Exceeded && state.Quota.NextRecoverAt.After(next) {
 									next = state.Quota.NextRecoverAt
 								}
@@ -894,7 +911,7 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
 									if otherState != nil && otherState != state {
 										otherState.Unavailable = true
 										otherState.Status = StatusError
-										otherQuotaNext := next
+										otherQuotaNext := credentialNext
 										if otherState.Quota.Exceeded && otherState.Quota.NextRecoverAt.After(otherQuotaNext) {
 											otherQuotaNext = otherState.Quota.NextRecoverAt
 										}
@@ -914,13 +931,15 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) {
 									}
 								}
 								auth.Unavailable = true
-								auth.Quota.Exceeded = true
-								auth.Quota.Reason = "credential_quota"
-								authNext := next
-								if auth.Quota.NextRecoverAt.After(authNext) {
+								authNext := credentialNext
+								if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" &&
+									auth.Quota.NextRecoverAt.After(authNext) {
 									authNext = auth.Quota.NextRecoverAt
 								}
+								auth.Quota.Exceeded = true
+								auth.Quota.Reason = "credential_quota"
 								auth.Quota.NextRecoverAt = authNext
+								auth.Quota.BackoffLevel = backoffLevel
 								auth.NextRetryAfter = authNext
 							}
 						case 408, 500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526:
--- a/sdk/cliproxy/auth/conductor_cooldown_monotonic_test.go
+++ b/sdk/cliproxy/auth/conductor_cooldown_monotonic_test.go
@@ -283,3 +283,172 @@ func TestManager_ApplyAuthFailureState_PreservesLongerCredentialDeadline(t *test
 		})
 	}
 }
+
+func TestManager_MarkResult_CredentialScopeDoesNotInheritModelQuotaDeadline(t *testing.T) {
+	withQuotaCooldownEnabled(t)
+
+	testCases := []struct {
+		name            string
+		credentialModel string
+	}{
+		{name: "cross_model", credentialModel: "claude-sonnet-5"},
+		{name: "same_model", credentialModel: "claude-fable-5-1"},
+	}
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			m, auth := newCooldownMonotonicManager(t, "claude-fable-5-1", "claude-sonnet-5", "claude-opus-5-5", "claude-sonnet-4")
+			ctx := context.Background()
+			for _, model := range []string{"claude-fable-5-1", "claude-sonnet-5", "claude-opus-5-5"} {
+				m.MarkResult(ctx, Result{AuthID: auth.ID, Provider: auth.Provider, Model: model, Success: true})
+			}
+
+			before := time.Now()
+			modelOnlyRetry := 8 * 24 * time.Hour
+			m.MarkResult(ctx, Result{
+				AuthID: auth.ID, Provider: auth.Provider, Model: "claude-fable-5-1",
+				RetryAfter: &modelOnlyRetry,
+				Error:      &Error{HTTPStatus: http.StatusTooManyRequests, Message: "Usage credits are required for this model."},
+			})
+
+			modelOnlyAuth, _ := m.GetByID(auth.ID)
+			if modelOnlyAuth.Quota.Reason != "quota" || !modelOnlyAuth.Quota.NextRecoverAt.After(before.Add(7*24*time.Hour)) {
+				t.Fatalf("precondition failed: model quota deadline was not aggregated: quota=%+v", modelOnlyAuth.Quota)
+			}
+			if blocked, _, _ := isAuthBlockedForModel(modelOnlyAuth, "claude-opus-5-5", time.Now()); blocked {
+				t.Fatal("model-only cooldown should not block an unrelated model")
+			}
+
+			credentialRetry := 3 * time.Hour
+			m.MarkResult(ctx, Result{
+				AuthID: auth.ID, Provider: auth.Provider, Model: testCase.credentialModel,
+				RetryAfter: &credentialRetry, CredentialScope: true,
+				Error: &Error{HTTPStatus: http.StatusTooManyRequests, Message: "shared window rejected"},
+			})
+
+			updated, _ := m.GetByID(auth.ID)
+			minCredentialDeadline := before.Add(credentialRetry - time.Hour)
+			maxCredentialDeadline := before.Add(credentialRetry + time.Hour)
+			if updated.Quota.Reason != "credential_quota" ||
+				updated.Quota.NextRecoverAt.Before(minCredentialDeadline) ||
+				updated.Quota.NextRecoverAt.After(maxCredentialDeadline) {
+				t.Fatalf("credential cooldown inherited a model-only deadline: reason=%q deadline=%v (in %v)",
+					updated.Quota.Reason, updated.Quota.NextRecoverAt, updated.Quota.NextRecoverAt.Sub(before))
+			}
+
+			opusBlocked, _, opusNext := isAuthBlockedForModel(updated, "claude-opus-5-5", before.Add(credentialRetry+time.Minute))
+			if opusBlocked {
+				t.Fatalf("opus remained blocked beyond credential cooldown: next=%v (in %v)", opusNext, opusNext.Sub(before))
+			}
+			fableBlocked, _, fableNext := isAuthBlockedForModel(updated, "claude-fable-5-1", before.Add(credentialRetry+time.Minute))
+			if !fableBlocked || !fableNext.After(before.Add(7*24*time.Hour)) {
+				t.Fatalf("model-only cooldown was not retained for Fable: blocked=%v next=%v", fableBlocked, fableNext)
+			}
+
+			credentialDeadline := updated.Quota.NextRecoverAt
+			shorterCredentialRetry := 30 * time.Minute
+			m.MarkResult(ctx, Result{
+				AuthID: auth.ID, Provider: auth.Provider, Model: "claude-sonnet-4",
+				RetryAfter: &shorterCredentialRetry, CredentialScope: true,
+				Error: &Error{HTTPStatus: http.StatusTooManyRequests, Message: "shorter shared window rejection"},
+			})
+			updated, _ = m.GetByID(auth.ID)
+			if !updated.Quota.NextRecoverAt.Equal(credentialDeadline) {
+				t.Fatalf("later credential-scoped cooldown shortened the active credential deadline: got=%v want=%v",
+					updated.Quota.NextRecoverAt, credentialDeadline)
+			}
+		})
+	}
+}
+
+func TestManager_MarkResult_CredentialScopeBackoffPersistsAcrossWindows(t *testing.T) {
+	withQuotaCooldownEnabled(t)
+
+	m, auth := newCooldownMonotonicManager(t, "model-a", "model-b")
+	ctx := context.Background()
+	m.MarkResult(ctx, Result{
+		AuthID: auth.ID, Provider: auth.Provider, Model: "model-a", CredentialScope: true,
+		Error: &Error{HTTPStatus: http.StatusTooManyRequests, Message: "credential quota exceeded"},
+	})
+
+	updated, _ := m.GetByID(auth.ID)
+	if updated.Quota.BackoffLevel != 1 {
+		t.Fatalf("first credential quota failure did not retain backoff level 1: quota=%+v", updated.Quota)
+	}
+
+	expired := time.Now().Add(-time.Minute)
+	m.mu.Lock()
+	storedAuth := m.auths[auth.ID]
+	storedAuth.Quota.NextRecoverAt = expired
+	storedAuth.NextRetryAfter = expired
+	for _, state := range storedAuth.ModelStates {
+		if state != nil {
+			state.Quota.NextRecoverAt = expired
+			state.NextRetryAfter = expired
+		}
+	}
+	m.mu.Unlock()
+
+	secondFailureAt := time.Now()
+	m.MarkResult(ctx, Result{
+		AuthID: auth.ID, Provider: auth.Provider, Model: "model-b", CredentialScope: true,
+		Error: &Error{HTTPStatus: http.StatusTooManyRequests, Message: "credential quota exceeded again"},
+	})
+
+	updated, _ = m.GetByID(auth.ID)
+	if updated.Quota.BackoffLevel != 2 {
+		t.Fatalf("second credential quota failure did not advance backoff to level 2: quota=%+v", updated.Quota)
+	}
+	if updated.Quota.NextRecoverAt.Before(secondFailureAt.Add(2 * quotaBackoffBase)) {
+		t.Fatalf("second credential quota failure did not use the next backoff window: deadline=%v", updated.Quota.NextRecoverAt)
+	}
+}
+
+func TestManager_MarkResult_CredentialScopeDoesNotInheritModelBackoffLevel(t *testing.T) {
+	withQuotaCooldownEnabled(t)
+
+	retryAfter := 10 * time.Second
+	tests := []struct {
+		name                 string
+		retryAfter           *time.Duration
+		wantBackoffLevel     int
+		wantCooldownDuration time.Duration
+	}{
+		{name: "without_retry_hint", wantBackoffLevel: 1, wantCooldownDuration: quotaBackoffBase},
+		{name: "with_retry_hint", retryAfter: &retryAfter, wantCooldownDuration: retryAfter},
+	}
+	for _, testCase := range tests {
+		t.Run(testCase.name, func(t *testing.T) {
+			m, auth := newCooldownMonotonicManager(t, "model-a", "model-b")
+			ctx := context.Background()
+			m.MarkResult(ctx, Result{
+				AuthID: auth.ID, Provider: auth.Provider, Model: "model-a",
+				Error: &Error{HTTPStatus: http.StatusTooManyRequests, Message: "model-only quota exceeded"},
+			})
+
+			modelOnlyAuth, _ := m.GetByID(auth.ID)
+			modelState := existingModelState(modelOnlyAuth, canonicalModelKey("model-a"))
+			if modelOnlyAuth.Quota.Reason != "quota" || modelState == nil || modelState.Quota.BackoffLevel != 1 {
+				t.Fatalf("precondition failed: model-only quota/backoff missing: auth=%+v state=%+v", modelOnlyAuth.Quota, modelState)
+			}
+
+			secondFailureStarted := time.Now()
+			m.MarkResult(ctx, Result{
+				AuthID: auth.ID, Provider: auth.Provider, Model: "model-a", RetryAfter: testCase.retryAfter,
+				CredentialScope: true,
+				Error:           &Error{HTTPStatus: http.StatusTooManyRequests, Message: "credential quota exceeded"},
+			})
+			secondFailureCompleted := time.Now()
+
+			updated, _ := m.GetByID(auth.ID)
+			if updated.Quota.BackoffLevel != testCase.wantBackoffLevel {
+				t.Fatalf("credential cooldown inherited model backoff: got level %d, want %d", updated.Quota.BackoffLevel, testCase.wantBackoffLevel)
+			}
+			minDeadline := secondFailureStarted.Add(testCase.wantCooldownDuration)
+			maxDeadline := secondFailureCompleted.Add(testCase.wantCooldownDuration)
+			if updated.Quota.NextRecoverAt.Before(minDeadline) || updated.Quota.NextRecoverAt.After(maxDeadline) {
+				t.Fatalf("credential cooldown deadline=%v, want between %v and %v",
+					updated.Quota.NextRecoverAt, minDeadline, maxDeadline)
+			}
+		})
+	}
+}