Files
CLIProxyAPI/sdk/cliproxy/auth/conductor_cooldown.go
Luis Pater d3e4db8b06 fix(antigravity): scope 403 validation and 401 inference errors to credential
- Mark Antigravity 403 VALIDATION_REQUIRED and 401 inference/auth errors as credential-scoped and close idle pooled transports on 403 TOS_VIOLATION and VALIDATION_REQUIRED.
- Propagate non-429 credential-scoped cooldowns to auth-level retry state and model projections so sibling models stop selecting blocked credentials.
- Separate refresh-clearable auth/grant cooldowns from non-auth credential cooldowns across OAuth refresh, token rotation, shadow unwrapping, and restart recovery.

Closes: #7814
2026-11-22 02:08:57 +08:00

4655 lines
160 KiB
Go

package auth
import (
"context"
"errors"
"fmt"
"io"
"maps"
"net"
"net/http"
"sort"
"strings"
"sync/atomic"
"syscall"
"time"
"github.com/MoeForge/moego/json"
"github.com/MoeForge/moego/regexp"
"github.com/gorilla/websocket"
"github.com/router-for-me/CLIProxyAPI/v8/internal/auth/codex"
"github.com/router-for-me/CLIProxyAPI/v8/internal/clienterror"
internalconfig "github.com/router-for-me/CLIProxyAPI/v8/internal/config"
internallogging "github.com/router-for-me/CLIProxyAPI/v8/internal/logging"
"github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
"github.com/router-for-me/CLIProxyAPI/v8/internal/thinking"
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
)
var quotaCooldownDisabled atomic.Bool
var transientErrorCooldownSeconds atomic.Int64
// SetQuotaCooldownDisabled toggles auth/model cooldown scheduling globally.
func SetQuotaCooldownDisabled(disable bool) {
quotaCooldownDisabled.Store(disable)
}
// SetTransientErrorCooldownSeconds configures cooldowns for 408/500/502/503/504.
// 0 keeps the legacy default; negative values disable transient error cooldowns.
func SetTransientErrorCooldownSeconds(seconds int) {
transientErrorCooldownSeconds.Store(int64(seconds))
}
// QuotaCooldownDisabledForAuth returns whether cooling is disabled for the auth under global settings.
func QuotaCooldownDisabledForAuth(auth *Auth) bool {
return quotaCooldownDisabledForAuth(auth)
}
// QuotaCooldownDisabledForAuthWithConfig returns whether cooling is disabled for the auth with the given config.
func QuotaCooldownDisabledForAuthWithConfig(auth *Auth, cfg *internalconfig.Config) bool {
return quotaCooldownDisabledForAuthWithConfig(auth, cfg)
}
func quotaCooldownDisabledForAuth(auth *Auth) bool {
return quotaCooldownDisabledForAuthWithConfig(auth, nil)
}
func quotaCooldownDisabledForAuthWithConfig(auth *Auth, cfg *internalconfig.Config) bool {
// Home owns cooldown state, so downstream instances must not schedule local cooldowns.
if cfg != nil && cfg.Home.Enabled {
return true
}
if auth != nil {
if override, ok := auth.DisableCoolingOverride(); ok {
return override
}
if override, ok := providerCoolingOverrideForAuth(auth, cfg); ok {
return override
}
}
if cfg != nil && cfg.DisableCooling {
return true
}
return quotaCooldownDisabled.Load()
}
func providerCoolingOverrideForAuth(auth *Auth, cfg *internalconfig.Config) (bool, bool) {
if auth == nil || cfg == nil {
return false, false
}
provider := strings.ToLower(strings.TrimSpace(auth.Provider))
if provider == "" {
return false, false
}
providerKey := ""
compatName := ""
if auth.Attributes != nil {
providerKey = strings.TrimSpace(auth.Attributes["provider_key"])
compatName = strings.TrimSpace(auth.Attributes["compat_name"])
}
if providerKey == "" && compatName == "" && provider != "openai-compatibility" {
return false, false
}
if providerKey == "" {
providerKey = provider
}
entry := resolveOpenAICompatConfig(cfg, providerKey, compatName, provider)
if entry == nil || entry.DisableCooling == nil {
return false, false
}
return *entry.DisableCooling, true
}
func nextTransientErrorRetryAfter(now time.Time) time.Time {
return recoverableFailureRetryAfterWithHint(now, nil, false)
}
func recoverableFailureRetryAfter(now time.Time, disableCooling bool) time.Time {
return recoverableFailureRetryAfterWithHint(now, nil, disableCooling)
}
// transientRateLimitRetryAfter resolves the model-scoped cooldown for a
// transient capacity 429. An explicit zero retry hint means the upstream
// accepts an immediate retry and suppresses the default transient cooldown;
// only a missing hint falls back to that default.
func transientRateLimitRetryAfter(now time.Time, retryAfter *time.Duration, disableCooling bool) time.Time {
if retryAfter != nil && *retryAfter == 0 {
return time.Time{}
}
return recoverableFailureRetryAfterWithHint(now, retryAfter, disableCooling)
}
func recoverableFailureRetryAfterWithHint(now time.Time, retryAfter *time.Duration, disableCooling bool) time.Time {
if disableCooling {
return time.Time{}
}
seconds := transientErrorCooldownSeconds.Load()
if seconds < 0 {
return time.Time{}
}
if retryAfter != nil && *retryAfter > 0 {
return now.Add(*retryAfter)
}
if seconds == 0 {
return now.Add(transientErrorCooldown)
}
return now.Add(time.Duration(seconds) * time.Second)
}
// SetConfig updates the runtime config snapshot used by request-time helpers.
// Callers should provide the latest config on reload so per-credential alias mapping stays in sync.
func (m *Manager) SetConfig(cfg *internalconfig.Config) {
if m == nil {
return
}
m.configCooldownMu.Lock()
defer m.configCooldownMu.Unlock()
if m.setConfigSnapshotLocked(cfg) {
m.persistCooldownStatesLocked(context.Background())
}
}
// SetConfigSnapshot updates only in-memory configuration state. It reports whether
// a caller must persist cleared cooldown state after its commit critical section.
func (m *Manager) SetConfigSnapshot(cfg *internalconfig.Config) bool {
if m == nil {
return false
}
m.configCooldownMu.Lock()
defer m.configCooldownMu.Unlock()
return m.setConfigSnapshotLocked(cfg)
}
func (m *Manager) setConfigSnapshotLocked(cfg *internalconfig.Config) bool {
if cfg == nil {
cfg = &internalconfig.Config{}
} else {
cfg = cfg.CloneForRuntime()
}
m.mu.RLock()
oldCooldownStore := m.cooldownStore
m.mu.RUnlock()
previousCfg, _ := m.runtimeConfig.Load().(*internalconfig.Config)
if homeSessionAliasTTL(previousCfg) != homeSessionAliasTTL(cfg) {
m.homeSessionAliases.clear()
}
m.runtimeConfig.Store(cfg)
clearedCooldowns := m.clearDisabledCooldownStates(cfg)
if clearedCooldowns && oldCooldownStore != nil {
m.mu.Lock()
if m.cooldownStore == oldCooldownStore {
m.pendingCooldownStateStore = oldCooldownStore
}
m.mu.Unlock()
}
if !cfg.Home.Enabled {
m.clearHomeRuntimeAuths()
}
m.rebuildAPIKeyModelAliasFromRuntimeConfig()
return clearedCooldowns
}
// ApplyConfigWithCooldownStateStore serializes a config update with its cooldown
// store transition. It persists the resulting state to the captured old store before
// exposing the resolved replacement store.
func (m *Manager) ApplyConfigWithCooldownStateStore(ctx context.Context, cfg *internalconfig.Config, store CooldownStateStore) bool {
if m == nil {
return false
}
if ctx == nil {
ctx = context.Background()
}
if errContext := ctx.Err(); errContext != nil {
return false
}
m.configCooldownMu.Lock()
defer m.configCooldownMu.Unlock()
m.mu.RLock()
oldStore := m.cooldownStore
m.mu.RUnlock()
m.setConfigSnapshotLocked(cfg)
if oldStore != nil && !m.persistCooldownStatesToLocked(ctx, oldStore) {
return false
}
if errContext := ctx.Err(); errContext != nil {
return false
}
m.mu.Lock()
defer m.mu.Unlock()
if m.cooldownStore != oldStore {
return false
}
if m.pendingCooldownStateStore == oldStore {
m.pendingCooldownStateStore = nil
}
m.cooldownStore = store
return true
}
// PersistCooldownStates writes the current cooldown snapshot using ctx.
func (m *Manager) PersistCooldownStates(ctx context.Context) {
m.persistCooldownStates(ctx)
}
// SwapCooldownStateStore persists cleared state to the old store before replacing it.
// Persistence is deliberately performed without holding the manager lock.
func (m *Manager) SwapCooldownStateStore(ctx context.Context, store CooldownStateStore, persistOld bool) bool {
if m == nil {
return false
}
if ctx == nil {
ctx = context.Background()
}
if errContext := ctx.Err(); errContext != nil {
return false
}
m.configCooldownMu.Lock()
defer m.configCooldownMu.Unlock()
m.mu.RLock()
oldStore := m.cooldownStore
pendingStore := m.pendingCooldownStateStore
m.mu.RUnlock()
storeToPersist := pendingStore
if storeToPersist == nil && persistOld {
storeToPersist = oldStore
}
if storeToPersist != nil && !m.persistCooldownStatesToLocked(ctx, storeToPersist) {
return false
}
if errContext := ctx.Err(); errContext != nil {
return false
}
m.mu.Lock()
defer m.mu.Unlock()
if m.cooldownStore != oldStore {
return false
}
if m.pendingCooldownStateStore == storeToPersist {
m.pendingCooldownStateStore = nil
}
m.cooldownStore = store
return true
}
func (m *Manager) cooldownDisabledForAuth(auth *Auth) bool {
if m == nil {
return quotaCooldownDisabledForAuth(auth)
}
cfg, _ := m.runtimeConfig.Load().(*internalconfig.Config)
return quotaCooldownDisabledForAuthWithConfig(auth, cfg)
}
func (m *Manager) clearDisabledCooldownStates(cfg *internalconfig.Config) bool {
if m == nil {
return false
}
now := time.Now()
snapshots := make([]*Auth, 0)
m.mu.Lock()
for _, auth := range m.auths {
if auth == nil {
continue
}
isDisabled := auth.Disabled || auth.Status == StatusDisabled
if !isDisabled && !quotaCooldownDisabledForAuthWithConfig(auth, cfg) {
continue
}
var cleared bool
if isDisabled {
cleared = clearCooldownStateForAuth(auth, now)
} else {
cleared = clearDisabledCooldownStateForAuth(auth, now)
}
if cleared {
snapshots = append(snapshots, auth.Clone())
}
}
m.mu.Unlock()
if m.scheduler != nil {
for _, snapshot := range snapshots {
m.scheduler.upsertAuth(snapshot)
}
}
return len(snapshots) > 0
}
// RestoreCooldownStates restores unexpired persisted cooldown records into registered auths.
func (m *Manager) RestoreCooldownStates(ctx context.Context) error {
if m == nil {
return nil
}
if ctx == nil {
ctx = context.Background()
}
m.mu.RLock()
store := m.cooldownStore
m.mu.RUnlock()
if store == nil {
return nil
}
records, errLoad := store.Load(ctx)
if errLoad != nil {
return errLoad
}
if len(records) == 0 {
return nil
}
now := time.Now()
authLevelRecords := make([]CooldownStateRecord, 0)
snapshotsByID := make(map[string]*Auth)
m.mu.Lock()
for _, record := range records {
if strings.TrimSpace(record.Model) == "" {
authLevelRecords = append(authLevelRecords, record)
continue
}
if m.restoreCooldownRecordLocked(record, now) {
if auth := m.auths[strings.TrimSpace(record.AuthID)]; auth != nil {
snapshotsByID[auth.ID] = auth.Clone()
}
}
}
for _, record := range authLevelRecords {
if m.restoreCooldownRecordLocked(record, now) {
if auth := m.auths[strings.TrimSpace(record.AuthID)]; auth != nil {
snapshotsByID[auth.ID] = auth.Clone()
}
}
}
for authID := range snapshotsByID {
if auth := m.auths[authID]; auth != nil && m.cooldownDisabledForAuth(auth) {
if clearDisabledCooldownStateForAuth(auth, now) {
snapshotsByID[authID] = auth.Clone()
}
}
}
m.mu.Unlock()
if m.scheduler != nil {
for _, snapshot := range snapshotsByID {
m.scheduler.upsertAuth(snapshot)
}
}
m.persistCooldownStates(context.Background())
return nil
}
func (m *Manager) restoreCooldownRecordLocked(record CooldownStateRecord, now time.Time) bool {
authID := strings.TrimSpace(record.AuthID)
if authID == "" || record.NextRetryAfter.IsZero() || !record.NextRetryAfter.After(now) {
return false
}
auth := m.auths[authID]
if auth == nil || auth.Disabled || auth.Status == StatusDisabled || (m.cooldownDisabledForAuth(auth) && !isForceCooldownRecordAt(record, now)) || hasUnauthorizedAuthFailure(auth) {
return false
}
updatedAt := record.UpdatedAt
if updatedAt.IsZero() {
updatedAt = now
}
reason := strings.TrimSpace(record.Reason)
model := strings.TrimSpace(record.Model)
quota := record.Quota
if quota.Exceeded && quota.NextRecoverAt.IsZero() {
quota.NextRecoverAt = record.NextRetryAfter
}
quota.cooldownEvaluated = quota.Exceeded
if executorKeyFromAuth(auth) == "claude" || strings.EqualFold(strings.TrimSpace(record.Provider), "claude") {
quota.cooldownSignals = sanitizePersistedClaudeCooldownSignals(record.CooldownSignals)
}
seedInitialForceCooldownDeadlines(auth)
outerForceUntil := recordOuterForceUntil(record)
indepForceUntil := recordIndependentForceUntil(record)
if model == "" {
prevNextRetry := auth.NextRetryAfter
prevQuota := auth.Quota.Clone()
prevUpdatedAt := auth.UpdatedAt
prevLastErr := cloneError(auth.LastError)
prevStatusMsg := auth.StatusMessage
hadLiveAuthBlock := auth.Unavailable && (prevNextRetry.After(now) || (prevQuota.Exceeded && prevQuota.NextRecoverAt.After(now)))
liveNewer := hadLiveAuthBlock && prevUpdatedAt.After(updatedAt)
auth.Unavailable = true
auth.Status = StatusError
if prevNextRetry.After(now) && prevNextRetry.After(record.NextRetryAfter) {
auth.NextRetryAfter = prevNextRetry
} else {
auth.NextRetryAfter = record.NextRetryAfter
}
mergedQuota := quota
prevCredQuota := prevQuota.Exceeded && strings.TrimSpace(prevQuota.Reason) == "credential_quota" && prevQuota.NextRecoverAt.After(now)
recCredQuota := quota.Exceeded && strings.TrimSpace(quota.Reason) == "credential_quota" && quota.NextRecoverAt.After(now)
if prevCredQuota || recCredQuota {
mergedQuota.Exceeded = true
mergedQuota.Reason = "credential_quota"
var credRecover time.Time
if prevCredQuota {
credRecover = prevQuota.NextRecoverAt
}
if recCredQuota && quota.NextRecoverAt.After(credRecover) {
credRecover = quota.NextRecoverAt
}
mergedQuota.NextRecoverAt = credRecover
if prevQuota.BackoffLevel > mergedQuota.BackoffLevel {
mergedQuota.BackoffLevel = prevQuota.BackoffLevel
}
if prevCredQuota && (!recCredQuota || prevQuota.NextRecoverAt.After(quota.NextRecoverAt) || len(mergedQuota.cooldownSignals) == 0) {
mergedQuota.cooldownSignals = prevQuota.cooldownSignals
}
} else if prevQuota.Exceeded && prevQuota.NextRecoverAt.After(now) {
mergedQuota.Exceeded = true
if prevQuota.NextRecoverAt.After(mergedQuota.NextRecoverAt) {
mergedQuota.NextRecoverAt = prevQuota.NextRecoverAt
}
if prevQuota.BackoffLevel > mergedQuota.BackoffLevel {
mergedQuota.BackoffLevel = prevQuota.BackoffLevel
}
if liveNewer && prevQuota.Reason != "" {
mergedQuota.Reason = prevQuota.Reason
} else if mergedQuota.Reason == "" {
mergedQuota.Reason = prevQuota.Reason
}
if prevQuota.NextRecoverAt.After(quota.NextRecoverAt) || len(mergedQuota.cooldownSignals) == 0 {
mergedQuota.cooldownSignals = prevQuota.cooldownSignals
}
}
if mergedQuota.Exceeded && !mergedQuota.NextRecoverAt.IsZero() && !mergedQuota.NextRecoverAt.After(now) {
mergedQuota = QuotaState{}
}
applyCooldownFields(&auth.Quota, mergedQuota)
auth.Quota = mergeQuotaObservation(auth.Quota, prevQuota)
auth.Quota = mergeQuotaObservation(auth.Quota, quota)
if credForce := recordCredQuotaForceUntil(record); credForce.After(now) && credForce.After(auth.credQuotaForceUntil) {
auth.credQuotaForceUntil = credForce
}
authForce := recordAuthLevelForceUntil(record)
if isLegacyAggregatedAuthRecord(record, auth.ModelStates, now) {
authForce = time.Time{}
}
if authForce.After(now) && authForce.After(auth.forceCooldownUntil) {
auth.forceCooldownUntil = authForce
}
if record.QuotaForceCooldownUntil.After(now) && record.QuotaForceCooldownUntil.After(auth.quotaForceCooldownUntil) {
auth.quotaForceCooldownUntil = record.QuotaForceCooldownUntil
}
if record.CredentialRetryAfter.After(now) && record.CredentialRetryAfter.After(auth.credentialRetryAfter) {
auth.credentialRetryAfter = record.CredentialRetryAfter
}
recordNonAuthRetry := record.NonAuthCredentialRetryAfter
if !record.NonAuthCredentialRetryEvaluated && recordNonAuthRetry.IsZero() && record.CredentialRetryAfter.After(now) && !isRefreshClearableCredentialError(record.LastError) {
recordNonAuthRetry = record.CredentialRetryAfter
}
recordNonAuthErr := record.NonAuthCredentialLastError
if recordNonAuthErr == nil && !isRefreshClearableCredentialError(record.LastError) {
recordNonAuthErr = record.LastError
}
if recordNonAuthRetry.After(now) {
if recordNonAuthRetry.After(auth.nonAuthCredentialRetryAfter) {
auth.nonAuthCredentialRetryAfter = recordNonAuthRetry
if recordNonAuthErr != nil {
auth.nonAuthCredentialLastError = cloneError(recordNonAuthErr)
}
} else if auth.nonAuthCredentialLastError == nil && recordNonAuthErr != nil {
auth.nonAuthCredentialLastError = cloneError(recordNonAuthErr)
}
}
if record.NonQuotaCooldownUntil.After(now) && record.NonQuotaCooldownUntil.After(auth.nonQuotaCooldownUntil) {
auth.nonQuotaCooldownUntil = record.NonQuotaCooldownUntil
}
auth.forceCooldownSeeded = true
auth.Generation++
if liveNewer {
auth.UpdatedAt = prevUpdatedAt
if prevStatusMsg != "" {
auth.StatusMessage = prevStatusMsg
} else if reason != "" {
auth.StatusMessage = reason
}
if prevLastErr != nil {
auth.LastError = prevLastErr
} else {
auth.LastError = cloneError(record.LastError)
}
} else {
auth.UpdatedAt = updatedAt
if reason != "" {
auth.StatusMessage = reason
}
auth.LastError = cloneError(record.LastError)
}
return true
}
state := ensureModelState(auth, model)
// Candidate independent layers for a wrapped record: the live shadow, or
// — when the live state is still unwrapped — the live state itself (a
// delayed store load must never erase a newer independent cooldown that
// was recorded after the snapshot was taken). A legacy pre-fix reporting
// state (reason "quota" whose own error is the explicit daily/weekly
// exhaustion message) is credential-derived and must not be captured;
// genuine model-scoped 429s carry different messages and stay eligible.
seedModelForceCooldownDeadline(state)
liveShadow := state.credentialQuotaShadow.cloneShallow()
if liveShadow == nil && !state.credentialQuotaPropagated && state.Quota.Reason != "credential_quota" &&
isModelStateActiveCooldown(state, now) && !isDevinQuotaExhaustionMessage(devinStateErrorText(state)) {
liveShadow = state.cloneShallow()
}
wasLiveWrapped := state.credentialQuotaPropagated || state.Quota.Reason == "credential_quota" ||
(auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now))
isWrappedRecord := record.IndependentState != nil || record.CredentialQuotaPropagated || strings.TrimSpace(quota.Reason) == "credential_quota"
var recordModelForceUntil time.Time
if !isWrappedRecord && outerForceUntil.After(now) {
recordModelForceUntil = outerForceUntil
} else if isWrappedRecord && outerForceUntil.After(now) && outerForceUntil.After(auth.credQuotaForceUntil) {
auth.credQuotaForceUntil = outerForceUntil
}
var recordQuotaForceUntil time.Time
if !isWrappedRecord && record.QuotaForceCooldownUntil.After(now) {
recordQuotaForceUntil = record.QuotaForceCooldownUntil
}
var recordNonQuotaUntil time.Time
if record.NonQuotaCooldownUntil.After(now) {
recordNonQuotaUntil = record.NonQuotaCooldownUntil
}
mergeModelState(state, &ModelState{
Unavailable: true,
Status: StatusError,
StatusMessage: reason,
NextRetryAfter: record.NextRetryAfter,
Quota: quota,
LastError: cloneError(record.LastError),
UpdatedAt: updatedAt,
forceCooldownUntil: recordModelForceUntil,
quotaForceCooldownUntil: recordQuotaForceUntil,
nonQuotaCooldownUntil: recordNonQuotaUntil,
forceCooldownSeeded: true,
})
// The persisted independent layer joins the merge newest-wins; a newer
// live independent layer (shadow or freshly captured unwrapped state,
// e.g. an own failure recorded during a delayed store load) is never
// clobbered by the older record. A wrapped record is any record whose
// outer reason is the propagated credential quota.
if !isWrappedRecord && wasLiveWrapped {
// An unwrapped independent record landing beneath a live wrap becomes
// the model's independent layer (unless it is itself a legacy
// credential-derived exhaustion state); the merged outer fields keep
// presenting the propagated credential quota.
if !isDevinQuotaExhaustionMessage(devinRecordErrorText(record)) {
recordShadow := &ModelState{
Unavailable: true,
Status: StatusError,
StatusMessage: reason,
NextRetryAfter: record.NextRetryAfter,
Quota: quota,
LastError: cloneError(record.LastError),
UpdatedAt: updatedAt,
forceCooldownUntil: recordModelForceUntil,
quotaForceCooldownUntil: recordQuotaForceUntil,
nonQuotaCooldownUntil: recordNonQuotaUntil,
forceCooldownSeeded: true,
}
preferred := state.credentialQuotaShadow.cloneShallow()
if liveShadow != nil {
preferred = mergeModelState(preferred, liveShadow.cloneShallow())
}
preferred = mergeModelState(preferred, recordShadow)
state.credentialQuotaShadow = preferred
}
state.credentialQuotaPropagated = true
applyCooldownFields(&state.Quota, QuotaState{
Exceeded: true,
Reason: "credential_quota",
NextRecoverAt: auth.Quota.NextRecoverAt,
BackoffLevel: auth.Quota.BackoffLevel,
cooldownSignals: auth.Quota.cooldownSignals,
})
if state.NextRetryAfter.Before(auth.Quota.NextRecoverAt) {
state.NextRetryAfter = auth.Quota.NextRecoverAt
}
if state.credentialQuotaShadow != nil && state.credentialQuotaShadow.NextRetryAfter.After(state.NextRetryAfter) {
state.NextRetryAfter = state.credentialQuotaShadow.NextRetryAfter
}
state.Unavailable = true
state.Status = StatusError
} else if isWrappedRecord {
state.credentialQuotaPropagated = true
var restoredShadow *ModelState
if record.IndependentState != nil {
restoredShadow = record.IndependentState.cloneShallow()
restoredShadow.Quota.cooldownEvaluated = restoredShadow.Quota.Exceeded
if indepForceUntil.After(restoredShadow.forceCooldownUntil) {
restoredShadow.forceCooldownUntil = indepForceUntil
}
if record.IndependentQuotaForceCooldownUntil.After(now) && record.IndependentQuotaForceCooldownUntil.After(restoredShadow.quotaForceCooldownUntil) {
restoredShadow.quotaForceCooldownUntil = record.IndependentQuotaForceCooldownUntil
}
if record.IndependentNonQuotaCooldownUntil.After(now) && record.IndependentNonQuotaCooldownUntil.After(restoredShadow.nonQuotaCooldownUntil) {
restoredShadow.nonQuotaCooldownUntil = record.IndependentNonQuotaCooldownUntil
}
restoredShadow.forceCooldownSeeded = true
if executorKeyFromAuth(auth) == "claude" || strings.EqualFold(strings.TrimSpace(record.Provider), "claude") {
restoredShadow.Quota.cooldownSignals = sanitizePersistedClaudeCooldownSignals(record.IndependentCooldownSignals)
}
}
state.credentialQuotaShadow = mergeModelState(liveShadow, restoredShadow)
if state.credentialQuotaShadow != nil && state.credentialQuotaShadow.NextRetryAfter.After(state.NextRetryAfter) {
state.NextRetryAfter = state.credentialQuotaShadow.NextRetryAfter
}
}
auth.forceCooldownSeeded = true
auth.Generation++
if updatedAt.After(auth.UpdatedAt) {
auth.UpdatedAt = updatedAt
}
updateAggregatedAvailability(auth, now)
return true
}
func isForceCooldownError(err *Error) bool {
return err != nil && err.Code == ErrorCodeForceCooldown
}
func recordOuterForceUntil(record CooldownStateRecord) time.Time {
if record.ForceCooldownEvaluated {
return record.ForceCooldownUntil
}
if isForceCooldownError(record.LastError) {
return record.NextRetryAfter
}
return time.Time{}
}
func recordCredQuotaForceUntil(record CooldownStateRecord) time.Time {
if record.ForceCooldownEvaluated {
return record.CredQuotaForceCooldownUntil
}
if strings.TrimSpace(record.Quota.Reason) == "credential_quota" && isForceCooldownError(record.LastError) {
if !record.Quota.NextRecoverAt.IsZero() {
return record.Quota.NextRecoverAt
}
return record.NextRetryAfter
}
return time.Time{}
}
func recordAuthLevelForceUntil(record CooldownStateRecord) time.Time {
if record.ForceCooldownEvaluated {
return record.ForceCooldownUntil
}
if strings.TrimSpace(record.Quota.Reason) != "credential_quota" && isForceCooldownError(record.LastError) {
return record.NextRetryAfter
}
return time.Time{}
}
// isLegacyAggregatedAuthRecord reports whether a pre-ForceCooldownEvaluated
// Model=="" record is merely the updateAggregatedAvailability copy of
// auth.ModelStates rather than an independent model-less cooldown.
func isLegacyAggregatedAuthRecord(record CooldownStateRecord, states map[string]*ModelState, now time.Time) bool {
if record.ForceCooldownEvaluated || len(states) == 0 || strings.TrimSpace(record.Quota.Reason) == "credential_quota" {
return false
}
var earliestRetry time.Time
matchedOrigin := false
hasState := false
for _, st := range states {
if st == nil {
continue
}
hasState = true
if st.Status == StatusDisabled {
continue
}
if !st.Unavailable || st.NextRetryAfter.IsZero() || !st.NextRetryAfter.After(now) {
return false
}
if earliestRetry.IsZero() || st.NextRetryAfter.Before(earliestRetry) {
earliestRetry = st.NextRetryAfter
}
if st.UpdatedAt.Equal(record.UpdatedAt) && cooldownErrorEqual(st.LastError, record.LastError) {
matchedOrigin = true
}
}
return hasState && matchedOrigin && !earliestRetry.IsZero() && record.NextRetryAfter.Equal(earliestRetry)
}
func recordIndependentForceUntil(record CooldownStateRecord) time.Time {
if record.IndependentState == nil {
return time.Time{}
}
if record.ForceCooldownEvaluated {
return record.IndependentForceCooldownUntil
}
if isForceCooldownError(record.IndependentState.LastError) {
return record.IndependentState.NextRetryAfter
}
return time.Time{}
}
func isForceCooldownRecordAt(record CooldownStateRecord, now time.Time) bool {
if strings.TrimSpace(record.Model) == "" {
return recordAuthLevelForceUntil(record).After(now) || recordCredQuotaForceUntil(record).After(now)
}
return recordOuterForceUntil(record).After(now) || recordIndependentForceUntil(record).After(now)
}
// Pure read-only force deadline accessors safe under RLock.
func effectiveCredQuotaForceUntil(auth *Auth) time.Time {
if auth == nil {
return time.Time{}
}
if auth.forceCooldownSeeded {
return auth.credQuotaForceUntil
}
if isForceCooldownError(auth.LastError) && auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" {
return auth.Quota.NextRecoverAt
}
return time.Time{}
}
func effectiveAuthForceUntil(auth *Auth) time.Time {
if auth == nil {
return time.Time{}
}
if auth.forceCooldownSeeded {
return auth.forceCooldownUntil
}
if len(auth.ModelStates) == 0 && isForceCooldownError(auth.LastError) {
return auth.NextRetryAfter
}
return time.Time{}
}
func effectiveModelForceUntil(state *ModelState) time.Time {
if state == nil {
return time.Time{}
}
if state.forceCooldownSeeded {
return state.forceCooldownUntil
}
isWrapped := state.credentialQuotaPropagated || state.Quota.Reason == "credential_quota"
if !isWrapped && isForceCooldownError(state.LastError) {
return state.NextRetryAfter
}
return time.Time{}
}
func seedModelForceCooldownDeadline(state *ModelState) {
if state == nil {
return
}
if state.credentialQuotaShadow != nil {
seedModelForceCooldownDeadline(state.credentialQuotaShadow)
}
if state.forceCooldownSeeded {
return
}
state.forceCooldownSeeded = true
isWrapped := state.credentialQuotaPropagated || state.Quota.Reason == "credential_quota"
if !isWrapped && isForceCooldownError(state.LastError) && !state.NextRetryAfter.IsZero() {
state.forceCooldownUntil = state.NextRetryAfter
}
}
// seedInitialForceCooldownDeadlines initializes explicit force-cooldown provenance
// once from pre-populated Auth/ModelState struct literals before runtime updates.
func seedInitialForceCooldownDeadlines(auth *Auth) {
if auth == nil {
return
}
for _, state := range auth.ModelStates {
seedModelForceCooldownDeadline(state)
}
if auth.forceCooldownSeeded {
return
}
auth.forceCooldownSeeded = true
if !isForceCooldownError(auth.LastError) {
return
}
if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && !auth.Quota.NextRecoverAt.IsZero() {
auth.credQuotaForceUntil = auth.Quota.NextRecoverAt
} else if len(auth.ModelStates) == 0 && !auth.NextRetryAfter.IsZero() {
auth.forceCooldownUntil = auth.NextRetryAfter
}
}
func clampStateToForceDeadline(state *ModelState, now time.Time) (kept bool, mutated bool) {
if state == nil {
return false, false
}
seedModelForceCooldownDeadline(state)
if !state.forceCooldownUntil.After(now) || !state.NextRetryAfter.After(now) {
return false, false
}
if state.NextRetryAfter.After(state.forceCooldownUntil) {
state.NextRetryAfter = state.forceCooldownUntil
mutated = true
}
if state.Quota.Exceeded && state.Quota.NextRecoverAt.After(state.forceCooldownUntil) {
state.Quota.NextRecoverAt = state.forceCooldownUntil
mutated = true
}
return true, mutated
}
// clearDisabledCooldownStateForAuth clears ordinary cooldown/quota state for an auth
// under disable-cooling while preserving active ErrorCodeForceCooldown deadlines.
func clearDisabledCooldownStateForAuth(auth *Auth, now time.Time) bool {
if auth == nil || hasUnauthorizedAuthFailure(auth) {
return false
}
seedInitialForceCooldownDeadlines(auth)
keepCredQuotaForce := auth.Quota.Exceeded &&
auth.Quota.Reason == "credential_quota" &&
auth.credQuotaForceUntil.After(now) &&
auth.Quota.NextRecoverAt.After(now)
keepAuthLevelForce := auth.Unavailable &&
auth.forceCooldownUntil.After(now) &&
auth.NextRetryAfter.After(now)
combinedAuthForceUntil := time.Time{}
if keepCredQuotaForce {
combinedAuthForceUntil = auth.credQuotaForceUntil
}
if keepAuthLevelForce && auth.forceCooldownUntil.After(combinedAuthForceUntil) {
combinedAuthForceUntil = auth.forceCooldownUntil
}
if len(auth.ModelStates) == 0 {
if keepCredQuotaForce || keepAuthLevelForce {
changed := false
if !keepCredQuotaForce && !auth.credQuotaForceUntil.IsZero() {
auth.credQuotaForceUntil = time.Time{}
changed = true
}
if !keepAuthLevelForce && !auth.forceCooldownUntil.IsZero() {
auth.forceCooldownUntil = time.Time{}
changed = true
}
if !auth.credentialRetryAfter.IsZero() || !auth.nonAuthCredentialRetryAfter.IsZero() {
auth.credentialRetryAfter = time.Time{}
auth.nonAuthCredentialRetryAfter = time.Time{}
auth.nonAuthCredentialLastError = nil
changed = true
}
if keepCredQuotaForce {
if !auth.Quota.NextRecoverAt.Equal(auth.credQuotaForceUntil) {
auth.Quota.NextRecoverAt = auth.credQuotaForceUntil
changed = true
}
} else if auth.Quota.Reason == "credential_quota" {
applyCooldownFields(&auth.Quota, QuotaState{})
changed = true
} else if auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(combinedAuthForceUntil) {
auth.Quota.NextRecoverAt = combinedAuthForceUntil
changed = true
}
if !auth.NextRetryAfter.Equal(combinedAuthForceUntil) {
auth.NextRetryAfter = combinedAuthForceUntil
changed = true
}
if changed {
auth.Generation++
auth.UpdatedAt = now
}
return changed
}
return clearCooldownStateForAuth(auth, now)
}
savedAuthNextRetry := combinedAuthForceUntil
savedAuthQuota := cooldownFieldsOf(auth.Quota)
if !keepCredQuotaForce {
if savedAuthQuota.Reason == "credential_quota" {
savedAuthQuota = QuotaState{}
} else if keepAuthLevelForce && savedAuthQuota.Exceeded && savedAuthQuota.NextRecoverAt.After(auth.forceCooldownUntil) {
savedAuthQuota.NextRecoverAt = auth.forceCooldownUntil
}
}
changed := false
if keepCredQuotaForce {
if !auth.Quota.NextRecoverAt.Equal(auth.credQuotaForceUntil) {
auth.Quota.NextRecoverAt = auth.credQuotaForceUntil
changed = true
}
if !auth.NextRetryAfter.Equal(combinedAuthForceUntil) {
auth.NextRetryAfter = combinedAuthForceUntil
changed = true
}
}
for _, state := range auth.ModelStates {
if state == nil {
continue
}
isWrapped := state.credentialQuotaPropagated || state.Quota.Reason == "credential_quota"
if isWrapped {
if keepCredQuotaForce {
var shadowDeadline time.Time
if state.credentialQuotaShadow != nil {
keptShadow, mutatedShadow := clampStateToForceDeadline(state.credentialQuotaShadow, now)
if !keptShadow {
state.credentialQuotaShadow = nil
state.UpdatedAt = now
changed = true
} else {
shadowDeadline = state.credentialQuotaShadow.NextRetryAfter
if mutatedShadow {
state.UpdatedAt = now
changed = true
}
}
}
effectiveOuter := auth.credQuotaForceUntil
if shadowDeadline.After(effectiveOuter) {
effectiveOuter = shadowDeadline
}
if state.Quota.NextRecoverAt.After(auth.credQuotaForceUntil) {
state.Quota.NextRecoverAt = auth.credQuotaForceUntil
state.UpdatedAt = now
changed = true
}
if state.NextRetryAfter.After(effectiveOuter) {
state.NextRetryAfter = effectiveOuter
state.UpdatedAt = now
changed = true
}
continue
}
if shadow := state.credentialQuotaShadow; shadow != nil {
if keptShadow, _ := clampStateToForceDeadline(shadow, now); keptShadow {
preservedSeq := shadow.cooldownRecordedSeq
*state = *shadow.cloneShallow()
state.cooldownRecordedSeq = preservedSeq
state.credentialQuotaPropagated = false
state.credentialQuotaShadow = nil
state.UpdatedAt = now
changed = true
continue
}
}
} else if keptModel, mutatedModel := clampStateToForceDeadline(state, now); keptModel {
if mutatedModel {
state.UpdatedAt = now
changed = true
}
continue
}
if state.Unavailable || !state.NextRetryAfter.IsZero() || state.Quota.Exceeded || !state.Quota.NextRecoverAt.IsZero() || state.credentialQuotaShadow != nil || state.credentialQuotaPropagated || !state.forceCooldownUntil.IsZero() {
state.Unavailable = false
state.NextRetryAfter = time.Time{}
applyCooldownFields(&state.Quota, QuotaState{})
state.cooldownRecordedSeq = 0
state.forceCooldownUntil = time.Time{}
state.quotaForceCooldownUntil = time.Time{}
state.nonQuotaCooldownUntil = time.Time{}
state.credentialQuotaShadow = nil
state.credentialQuotaPropagated = false
state.UpdatedAt = now
changed = true
}
}
if !keepCredQuotaForce {
auth.credQuotaForceUntil = time.Time{}
}
if !keepAuthLevelForce {
auth.forceCooldownUntil = time.Time{}
auth.quotaForceCooldownUntil = time.Time{}
}
if !auth.credentialRetryAfter.IsZero() || !auth.nonAuthCredentialRetryAfter.IsZero() {
auth.credentialRetryAfter = time.Time{}
auth.nonAuthCredentialRetryAfter = time.Time{}
auth.nonAuthCredentialLastError = nil
changed = true
}
if !keepAuthLevelForce && !auth.nonQuotaCooldownUntil.IsZero() {
auth.nonQuotaCooldownUntil = time.Time{}
}
prevUnavailable := auth.Unavailable
prevNextRetry := auth.NextRetryAfter
prevQuota := cooldownFieldsOf(auth.Quota)
if !keepCredQuotaForce {
auth.Unavailable = false
auth.NextRetryAfter = time.Time{}
applyCooldownFields(&auth.Quota, QuotaState{})
if !keepAuthLevelForce {
auth.cooldownRecordedSeq = 0
}
updateAggregatedAvailability(auth, now)
}
if keepAuthLevelForce {
auth.Unavailable = true
if savedAuthNextRetry.After(auth.NextRetryAfter) {
auth.NextRetryAfter = savedAuthNextRetry
}
if !keepCredQuotaForce && savedAuthQuota.Exceeded && savedAuthQuota.NextRecoverAt.After(now) && !auth.Quota.NextRecoverAt.After(savedAuthQuota.NextRecoverAt) {
applyCooldownFields(&auth.Quota, savedAuthQuota)
}
}
if auth.Unavailable != prevUnavailable || !auth.NextRetryAfter.Equal(prevNextRetry) || !cooldownQuotaEqual(cooldownFieldsOf(auth.Quota), prevQuota) {
changed = true
}
if changed {
auth.Generation++
auth.UpdatedAt = now
}
return changed
}
func clearCooldownStateForAuth(auth *Auth, now time.Time) bool {
if auth == nil {
return false
}
if hasUnauthorizedAuthFailure(auth) {
return false
}
isDisabled := auth.Disabled || auth.Status == StatusDisabled
changed := false
auth.forceCooldownUntil = time.Time{}
auth.quotaForceCooldownUntil = time.Time{}
auth.credQuotaForceUntil = time.Time{}
auth.credentialRetryAfter = time.Time{}
auth.nonAuthCredentialRetryAfter = time.Time{}
auth.nonAuthCredentialLastError = nil
auth.nonQuotaCooldownUntil = time.Time{}
auth.cooldownRecordedSeq = 0
if isDisabled {
if !auth.NextRetryAfter.IsZero() || auth.Quota.Exceeded || !auth.Quota.NextRecoverAt.IsZero() {
auth.NextRetryAfter = time.Time{}
applyCooldownFields(&auth.Quota, QuotaState{})
auth.UpdatedAt = now
changed = true
}
} else if auth.Unavailable || !auth.NextRetryAfter.IsZero() || auth.Quota.Exceeded || !auth.Quota.NextRecoverAt.IsZero() {
auth.Unavailable = false
auth.NextRetryAfter = time.Time{}
applyCooldownFields(&auth.Quota, QuotaState{})
auth.UpdatedAt = now
changed = true
}
for _, state := range auth.ModelStates {
if state == nil {
continue
}
state.forceCooldownUntil = time.Time{}
state.quotaForceCooldownUntil = time.Time{}
state.nonQuotaCooldownUntil = time.Time{}
state.cooldownRecordedSeq = 0
if state.Status == StatusDisabled {
if !state.NextRetryAfter.IsZero() || state.Quota.Exceeded || !state.Quota.NextRecoverAt.IsZero() || state.credentialQuotaShadow != nil || state.credentialQuotaPropagated {
state.NextRetryAfter = time.Time{}
applyCooldownFields(&state.Quota, QuotaState{})
state.credentialQuotaShadow = nil
state.credentialQuotaPropagated = false
state.UpdatedAt = now
changed = true
}
} else if state.Unavailable || !state.NextRetryAfter.IsZero() || state.Quota.Exceeded || !state.Quota.NextRecoverAt.IsZero() || state.credentialQuotaShadow != nil || state.credentialQuotaPropagated {
state.Unavailable = false
state.NextRetryAfter = time.Time{}
applyCooldownFields(&state.Quota, QuotaState{})
state.credentialQuotaShadow = nil
state.credentialQuotaPropagated = false
state.UpdatedAt = now
changed = true
}
}
if len(auth.ModelStates) > 0 {
updateAggregatedAvailability(auth, now)
}
if changed {
auth.Generation++
auth.UpdatedAt = now
}
return changed
}
func dedupeStrings(values []string) []string {
if len(values) < 2 {
return values
}
seen := make(map[string]struct{}, len(values))
out := values[:0]
for _, value := range values {
value = strings.TrimSpace(value)
if value == "" {
continue
}
if _, ok := seen[value]; ok {
continue
}
seen[value] = struct{}{}
out = append(out, value)
}
return out
}
// ResetQuota clears quota/cooldown state for an auth and resumes registry routing.
func (m *Manager) ResetQuota(ctx context.Context, authID string) (*Auth, []string, error) {
if m == nil {
return nil, nil, nil
}
authID = strings.TrimSpace(authID)
if authID == "" {
return nil, nil, fmt.Errorf("auth id is required")
}
now := time.Now()
var snapshot *Auth
models := make([]string, 0)
registeredModels := modelsForRegisteredAuth(authID)
cooldownStateChanged := false
releaseMutation := m.lockAuthMutation(authID)
defer releaseMutation()
m.mu.Lock()
auth, ok := m.auths[authID]
if !ok || auth == nil {
m.mu.Unlock()
return nil, nil, nil
}
var cooldownRecordsBefore []CooldownStateRecord
trackCooldownState := m.cooldownStore != nil
if trackCooldownState {
cooldownRecordsBefore = m.cooldownStateRecordsForAuthLocked(auth, now)
}
for modelKey, state := range auth.ModelStates {
if strings.TrimSpace(modelKey) == "" {
continue
}
models = append(models, modelKey)
if state != nil {
resetModelState(state, now)
}
}
if clearCooldownStateForAuth(auth, now) {
if len(models) == 0 {
models = append(models, registeredModels...)
}
} else if len(auth.ModelStates) > 0 {
updateAggregatedAvailability(auth, now)
}
if len(models) == 0 {
models = append(models, registeredModels...)
}
models = dedupeStrings(models)
if !auth.Disabled && auth.Status != StatusDisabled && !hasModelError(auth, now) {
if !hasUnauthorizedAuthFailure(auth) {
auth.LastError = nil
auth.StatusMessage = ""
auth.Status = StatusActive
}
}
auth.Generation++
auth.UpdatedAt = now
snapshot = auth.Clone()
if trackCooldownState {
cooldownRecordsAfter := m.cooldownStateRecordsForAuthLocked(auth, now)
cooldownStateChanged = !cooldownStateRecordsEqual(cooldownRecordsBefore, cooldownRecordsAfter)
}
errPersist := m.persistLocked(ctx, auth)
m.mu.Unlock()
releaseMutation()
defer func() {
if cooldownStateChanged {
m.persistCooldownStates(context.Background())
}
}()
supportedModels, regEpoch := registry.GetGlobalRegistry().GetModelsAndEpochForClient(authID)
projections := make([]registry.ClientModelProjection, 0, len(supportedModels))
for _, sm := range supportedModels {
if sm == nil || strings.TrimSpace(sm.ID) == "" {
continue
}
projections = append(projections, m.clientModelProjectionForAuth(snapshot, sm.ID, now))
}
if snapshot != nil {
registry.GetGlobalRegistry().ApplyClientModelProjections(authID, regEpoch, snapshot.Generation, projections)
}
if m.scheduler != nil && snapshot != nil {
m.scheduler.upsertAuth(snapshot)
}
if errPersist != nil {
return nil, nil, errPersist
}
return snapshot, models, nil
}
func modelsForRegisteredAuth(authID string) []string {
supportedModels := registry.GetGlobalRegistry().GetModelsForClient(authID)
models := make([]string, 0, len(supportedModels))
for _, supportedModel := range supportedModels {
if supportedModel == nil || strings.TrimSpace(supportedModel.ID) == "" {
continue
}
models = append(models, canonicalModelKey(supportedModel.ID))
}
return models
}
func (m *Manager) persistCooldownStates(ctx context.Context) {
if m == nil {
return
}
m.configCooldownMu.Lock()
defer m.configCooldownMu.Unlock()
m.persistCooldownStatesLocked(ctx)
}
func (m *Manager) persistCooldownStatesLocked(ctx context.Context) {
m.mu.RLock()
store := m.cooldownStore
m.mu.RUnlock()
if m.persistCooldownStatesToLocked(ctx, store) {
m.mu.Lock()
if m.pendingCooldownStateStore == store {
m.pendingCooldownStateStore = nil
}
m.mu.Unlock()
}
}
func (m *Manager) persistCooldownStatesToLocked(ctx context.Context, store CooldownStateStore) bool {
if m == nil || store == nil {
return true
}
if ctx == nil {
ctx = context.Background()
}
if errContext := ctx.Err(); errContext != nil {
return false
}
records := m.cooldownStateRecordsSnapshot()
if errSave := store.Save(ctx, records); errSave != nil {
logEntryWithRequestID(ctx).Warnf("failed to persist cooldown state: %v", errSave)
return false
}
return ctx.Err() == nil
}
func (m *Manager) cooldownStateRecordsSnapshot() []CooldownStateRecord {
now := time.Now()
records := make([]CooldownStateRecord, 0)
m.mu.RLock()
for _, auth := range m.auths {
records = append(records, m.cooldownStateRecordsForAuthLocked(auth, now)...)
}
m.mu.RUnlock()
sort.Slice(records, func(i, j int) bool {
if records[i].Provider != records[j].Provider {
return records[i].Provider < records[j].Provider
}
if records[i].AuthID != records[j].AuthID {
return records[i].AuthID < records[j].AuthID
}
return records[i].Model < records[j].Model
})
return records
}
func (m *Manager) cooldownStateRecordsForAuthLocked(auth *Auth, now time.Time) []CooldownStateRecord {
if auth == nil || auth.ID == "" || auth.Disabled || auth.Status == StatusDisabled {
return nil
}
coolingDisabled := m.cooldownDisabledForAuth(auth)
records := make([]CooldownStateRecord, 0, 1+len(auth.ModelStates))
if record, ok := authCooldownStateRecord(auth, now); ok && (!coolingDisabled || isForceCooldownRecordAt(record, now)) {
records = append(records, record)
}
for model, state := range auth.ModelStates {
if record, ok := modelCooldownStateRecord(auth, model, state, now); ok && (!coolingDisabled || isForceCooldownRecordAt(record, now)) {
records = append(records, record)
}
}
sort.Slice(records, func(i, j int) bool {
return records[i].Model < records[j].Model
})
return records
}
func cooldownStateRecordsEqual(a, b []CooldownStateRecord) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if !cooldownStateRecordEqual(a[i], b[i]) {
return false
}
}
return true
}
func cooldownStateRecordEqual(a, b CooldownStateRecord) bool {
if a.CredentialQuotaPropagated != b.CredentialQuotaPropagated {
return false
}
if a.Provider != b.Provider ||
a.AuthID != b.AuthID ||
a.AuthFile != b.AuthFile ||
a.Model != b.Model ||
a.Status != b.Status ||
a.Reason != b.Reason ||
!a.NextRetryAfter.Equal(b.NextRetryAfter) ||
!a.UpdatedAt.Equal(b.UpdatedAt) ||
!cooldownQuotaEqual(a.Quota, b.Quota) {
return false
}
if !cooldownErrorEqual(a.LastError, b.LastError) {
return false
}
if a.ForceCooldownEvaluated != b.ForceCooldownEvaluated ||
!a.ForceCooldownUntil.Equal(b.ForceCooldownUntil) ||
!a.CredQuotaForceCooldownUntil.Equal(b.CredQuotaForceCooldownUntil) ||
!a.IndependentForceCooldownUntil.Equal(b.IndependentForceCooldownUntil) ||
!a.CredentialRetryAfter.Equal(b.CredentialRetryAfter) ||
a.NonAuthCredentialRetryEvaluated != b.NonAuthCredentialRetryEvaluated ||
!a.NonAuthCredentialRetryAfter.Equal(b.NonAuthCredentialRetryAfter) ||
!cooldownErrorEqual(a.NonAuthCredentialLastError, b.NonAuthCredentialLastError) {
return false
}
if !maps.Equal(a.CooldownSignals, b.CooldownSignals) || !maps.Equal(a.IndependentCooldownSignals, b.IndependentCooldownSignals) {
return false
}
return cooldownIndependentStateEqual(a.IndependentState, b.IndependentState)
}
func cooldownIndependentStateEqual(a, b *ModelState) bool {
if a == nil || b == nil {
return a == b
}
return a.Status == b.Status &&
a.StatusMessage == b.StatusMessage &&
a.Unavailable == b.Unavailable &&
a.NextRetryAfter.Equal(b.NextRetryAfter) &&
a.UpdatedAt.Equal(b.UpdatedAt) &&
cooldownQuotaEqual(a.Quota, b.Quota) &&
cooldownErrorEqual(a.LastError, b.LastError)
}
func cooldownQuotaEqual(a, b QuotaState) bool {
return a.Exceeded == b.Exceeded &&
a.Reason == b.Reason &&
a.BackoffLevel == b.BackoffLevel &&
a.NextRecoverAt.Equal(b.NextRecoverAt)
}
func cooldownErrorEqual(a, b *Error) bool {
if a == nil || b == nil {
return a == b
}
return a.Code == b.Code &&
a.Message == b.Message &&
a.Retryable == b.Retryable &&
a.HTTPStatus == b.HTTPStatus &&
maps.Equal(a.Metadata, b.Metadata)
}
func authCooldownStateRecord(auth *Auth, now time.Time) (CooldownStateRecord, bool) {
if auth == nil || !auth.Unavailable {
return CooldownStateRecord{}, false
}
var authForceUntil time.Time
if u := effectiveAuthForceUntil(auth); u.After(now) {
authForceUntil = u
}
var credentialRetryAfter time.Time
if !auth.credentialRetryAfter.IsZero() && auth.credentialRetryAfter.After(now) {
credentialRetryAfter = auth.credentialRetryAfter
}
var nonAuthCredentialRetryAfter time.Time
var nonAuthCredentialLastError *Error
if !auth.nonAuthCredentialRetryAfter.IsZero() && auth.nonAuthCredentialRetryAfter.After(now) {
nonAuthCredentialRetryAfter = auth.nonAuthCredentialRetryAfter
nonAuthCredentialLastError = cloneError(auth.nonAuthCredentialLastError)
}
effectiveNextRetry := auth.NextRetryAfter
if auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(effectiveNextRetry) {
effectiveNextRetry = auth.Quota.NextRecoverAt
}
if authForceUntil.After(effectiveNextRetry) {
effectiveNextRetry = authForceUntil
}
if credentialRetryAfter.After(effectiveNextRetry) {
effectiveNextRetry = credentialRetryAfter
}
if effectiveNextRetry.IsZero() || !effectiveNextRetry.After(now) {
return CooldownStateRecord{}, false
}
var cooldownSignals map[string]string
if auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(now) && !effectiveNextRetry.After(auth.Quota.NextRecoverAt) {
cooldownSignals = sanitizePersistedClaudeCooldownSignals(auth.Quota.cooldownSignals)
}
var credQuotaForceUntil time.Time
if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now) {
if u := effectiveCredQuotaForceUntil(auth); u.After(now) {
credQuotaForceUntil = u
}
}
var authQuotaForceUntil time.Time
if auth.quotaForceCooldownUntil.After(now) {
authQuotaForceUntil = auth.quotaForceCooldownUntil
}
var nonQuotaUntil time.Time
if auth.nonQuotaCooldownUntil.After(now) {
nonQuotaUntil = auth.nonQuotaCooldownUntil
}
return CooldownStateRecord{
Provider: strings.TrimSpace(auth.Provider),
AuthID: auth.ID,
AuthFile: cooldownAuthFile(auth),
Status: "cooling",
NextRetryAfter: effectiveNextRetry,
Reason: cooldownReason(auth.StatusMessage, auth.Quota, auth.LastError),
Quota: cooldownFieldsOf(auth.Quota),
LastError: cloneError(auth.LastError),
UpdatedAt: auth.UpdatedAt,
CooldownSignals: cooldownSignals,
ForceCooldownUntil: authForceUntil,
QuotaForceCooldownUntil: authQuotaForceUntil,
CredQuotaForceCooldownUntil: credQuotaForceUntil,
ForceCooldownEvaluated: true,
CredentialRetryAfter: credentialRetryAfter,
NonAuthCredentialRetryAfter: nonAuthCredentialRetryAfter,
NonAuthCredentialLastError: nonAuthCredentialLastError,
NonAuthCredentialRetryEvaluated: true,
NonQuotaCooldownUntil: nonQuotaUntil,
}, true
}
func modelCooldownStateRecord(auth *Auth, model string, state *ModelState, now time.Time) (CooldownStateRecord, bool) {
model = strings.TrimSpace(model)
if auth == nil || state == nil || model == "" || !state.Unavailable || state.NextRetryAfter.IsZero() || !state.NextRetryAfter.After(now) {
return CooldownStateRecord{}, false
}
var cooldownSignals map[string]string
liveCredentialWrap := auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now)
expiredCredentialWrap := (state.credentialQuotaPropagated || state.Quota.Reason == "credential_quota") && !liveCredentialWrap
if !expiredCredentialWrap && state.Quota.Exceeded && state.Quota.NextRecoverAt.After(now) &&
(state.Quota.Reason == "credential_quota" || !state.NextRetryAfter.After(state.Quota.NextRecoverAt)) {
cooldownSignals = sanitizePersistedClaudeCooldownSignals(state.Quota.cooldownSignals)
}
var outerForceUntil time.Time
if liveCredentialWrap {
if u := effectiveCredQuotaForceUntil(auth); u.After(now) {
outerForceUntil = u
}
} else if u := effectiveModelForceUntil(state); u.After(now) {
outerForceUntil = u
}
var modelQuotaForceUntil time.Time
if !liveCredentialWrap && state.quotaForceCooldownUntil.After(now) {
modelQuotaForceUntil = state.quotaForceCooldownUntil
}
var modelNonQuotaUntil time.Time
if state.nonQuotaCooldownUntil.After(now) {
modelNonQuotaUntil = state.nonQuotaCooldownUntil
}
record := CooldownStateRecord{
Provider: strings.TrimSpace(auth.Provider),
AuthID: auth.ID,
AuthFile: cooldownAuthFile(auth),
Model: model,
Status: "cooling",
NextRetryAfter: state.NextRetryAfter,
Reason: cooldownReason(state.StatusMessage, state.Quota, state.LastError),
Quota: cooldownFieldsOf(state.Quota),
LastError: cloneError(state.LastError),
UpdatedAt: state.UpdatedAt,
CooldownSignals: cooldownSignals,
ForceCooldownUntil: outerForceUntil,
QuotaForceCooldownUntil: modelQuotaForceUntil,
ForceCooldownEvaluated: true,
NonQuotaCooldownUntil: modelNonQuotaUntil,
}
if state.credentialQuotaShadow != nil {
record.IndependentState = state.credentialQuotaShadow.cloneShallow()
if u := effectiveModelForceUntil(state.credentialQuotaShadow); u.After(now) {
record.IndependentForceCooldownUntil = u
}
if state.credentialQuotaShadow.quotaForceCooldownUntil.After(now) {
record.IndependentQuotaForceCooldownUntil = state.credentialQuotaShadow.quotaForceCooldownUntil
}
if state.credentialQuotaShadow.nonQuotaCooldownUntil.After(now) {
record.IndependentNonQuotaCooldownUntil = state.credentialQuotaShadow.nonQuotaCooldownUntil
}
if state.credentialQuotaShadow.Quota.Exceeded && state.credentialQuotaShadow.Quota.NextRecoverAt.After(now) &&
!state.credentialQuotaShadow.NextRetryAfter.After(state.credentialQuotaShadow.Quota.NextRecoverAt) {
record.IndependentCooldownSignals = sanitizePersistedClaudeCooldownSignals(state.credentialQuotaShadow.Quota.cooldownSignals)
}
}
record.CredentialQuotaPropagated = state.credentialQuotaPropagated
return record, true
}
func cooldownReason(statusMessage string, quota QuotaState, lastErr *Error) string {
if reason := strings.TrimSpace(quota.Reason); reason != "" {
return reason
}
if statusMessage = strings.TrimSpace(statusMessage); statusMessage != "" {
return statusMessage
}
if lastErr != nil {
if code := strings.TrimSpace(lastErr.Code); code != "" {
return code
}
if message := strings.TrimSpace(lastErr.Message); message != "" {
return message
}
}
return ""
}
// nextExecutionSeq stamps a new upstream dispatch with the next monotonic
// execution sequence. The stamp travels on the execution's Result so that
// MarkResult can order the execution against cooldowns recorded meanwhile.
// NextExecutionSeq allocates the next monotonic execution sequence number for
// ordering concurrent MarkResult updates against newly recorded cooldowns.
func (m *Manager) NextExecutionSeq() uint64 {
return m.nextExecutionSeq()
}
func (m *Manager) nextExecutionSeq() uint64 {
if m == nil {
return 0
}
return m.executionSeq.Add(1)
}
// currentExecutionSeq returns the latest execution sequence issued so far.
// Capturing it as the cooldown watermark when a cooldown is recorded marks
// every execution dispatched up to that moment as predating the cooldown.
func (m *Manager) currentExecutionSeq() uint64 {
if m == nil {
return 0
}
return m.executionSeq.Load()
}
// isStaleExecutionResult reports whether an execution result originated from a superseded
// credential version or an earlier registration epoch than the currently active auth.
func isStaleExecutionResult(result Result, current *Auth) bool {
if current == nil {
return false
}
staleEpoch := result.RegistrationEpoch > 0 && result.RegistrationEpoch < current.RegistrationEpoch
if staleEpoch {
return true
}
staleVersion := result.CredentialVersion < current.CredentialVersion && (result.CredentialVersion > 0 || current.CredentialVersion > 1)
if staleVersion && result.Error != nil &&
(strings.EqualFold(strings.TrimSpace(result.Provider), "antigravity") || strings.EqualFold(strings.TrimSpace(current.Provider), "antigravity")) &&
isAntigravityValidationRequiredError(result.Error) {
// An Antigravity account validation challenge applies to the account, so a
// plain OAuth access-token refresh (same credential, identical refresh token)
// must not discard the signal. Conversely, a stale-version validation result
// must only be accepted when both refresh tokens are non-empty and equal, so
// replacing refresh_token (even if access_token temporarily matches or is
// empty) never carries an old account's challenge onto the new credential.
refreshNow := authRefreshToken(current)
if refreshNow != "" && result.RefreshToken != "" && refreshNow == result.RefreshToken {
return false
}
return true
}
if result.AccessToken != "" && authMatchesTokenPartition(current, result.AccessToken) {
return false
}
if isTOSViolationResultError(result.Error) {
// A service-scoped TOS suspension applies to the account, so a plain OAuth
// access-token refresh (same credential, refresh token retained) must not
// discard the terminal signal. A genuine credential replacement under the
// same auth ID is a different account: an old request's TOS result must
// not disable the fresh credential. The access token alone cannot tell
// these apart, so exempt only when the refresh credential proves the same
// account (identical refresh token) or the result carries no account hint
// (internal/token-less paths where replacement cannot be detected).
if result.AccessToken == "" {
return false
}
refreshNow := authRefreshToken(current)
if refreshNow != "" && result.RefreshToken != "" && refreshNow == result.RefreshToken {
return false
}
}
return staleVersion
}
// MarkResult records an execution result and notifies hooks.
func (m *Manager) MarkResult(ctx context.Context, result Result) {
if result.AuthID == "" {
return
}
if ctx == nil {
ctx = context.Background()
}
if m != nil {
if policy := m.ResultPolicy(); policy != nil {
result = policy.ApplyResultPolicy(ctx, result)
if result.AuthID == "" {
return
}
}
}
modelKey := canonicalModelKey(result.Model)
var authSnapshot *Auth
cooldownStateChanged := false
scheduleUnauthorizedRefresh := false
now := time.Now()
// A 401 condemns the access token, except one a -and-cooldown rule matched: that only cools the credential.
rejectsToken := !result.Success && (result.Error == nil || result.Error.Code != ErrorCodeForceCooldown) && !shouldSkipCredentialCooldown(result.Error) && (statusCodeFromResult(result.Error) == http.StatusUnauthorized || isUnauthorizedError(result.Error))
releaseMutation := m.lockAuthMutation(result.AuthID)
defer releaseMutation()
m.mu.Lock()
if auth, ok := m.auths[result.AuthID]; ok && auth != nil {
if isStaleExecutionResult(result, auth) {
staleEpoch := result.RegistrationEpoch > 0 && result.RegistrationEpoch < auth.RegistrationEpoch
if !staleEpoch && rejectsToken && result.AccessToken != "" {
if recordRejectedTokenOnAuth(auth, result.AccessToken) {
auth.Generation++
auth.UpdatedAt = now
}
}
m.mu.Unlock()
releaseMutation()
m.hook.OnResult(ctx, result)
m.publishErrorEvent(result, nil)
return
}
if modelKey == "" && strings.TrimSpace(result.RouteModel) != "" {
if m != nil {
modelKey = m.selectionModelKeyForAuth(auth, result.RouteModel)
}
if modelKey == "" {
modelKey = canonicalModelKey(result.RouteModel)
}
}
now = time.Now()
responseHeaders := internallogging.GetResponseHeaders(ctx)
var cooldownRecordsBefore []CooldownStateRecord
trackCooldownState := m.cooldownStore != nil
if trackCooldownState {
cooldownRecordsBefore = m.cooldownStateRecordsForAuthLocked(auth, now)
}
if m != nil {
_ = m.migrateAuthAliasModelStatesLocked(auth, nil, now, false)
if modelKey != "" {
modelKey = m.resolveMarkResultModelKeyLocked(auth, modelKey, result.RouteModel)
}
}
modelState := existingModelState(auth, modelKey)
auth.recordRecentRequest(now, result.Success)
if result.Success {
auth.Success++
} else {
auth.Failed++
}
wasTerminalUnauthorized := hasUnauthorizedAuthFailure(auth)
// retainModelCooldown pins a still-live model cooldown against a success
// from an execution dispatched before that cooldown was recorded. Such a
// success predates the failure and proves nothing about recovery, so the
// state transition it normally causes is skipped entirely.
retainModelCooldown := modelKey != "" && !wasTerminalUnauthorized && result.ExecutionSeq > 0
if retainModelCooldown {
if existing := existingModelState(auth, modelKey); existing != nil && isModelStateActiveCooldown(existing, now) &&
existing.cooldownRecordedSeq > 0 && result.ExecutionSeq <= existing.cooldownRecordedSeq {
modelState = existing
} else {
retainModelCooldown = false
}
}
retainAuthCooldown := modelKey == "" && !wasTerminalUnauthorized && result.ExecutionSeq > 0 &&
isAuthActiveCooldown(auth, now) && auth.cooldownRecordedSeq > 0 && result.ExecutionSeq <= auth.cooldownRecordedSeq
retainCredentialRetry := !wasTerminalUnauthorized && result.ExecutionSeq > 0 &&
!auth.credentialRetryAfter.IsZero() && auth.credentialRetryAfter.After(now) &&
auth.cooldownRecordedSeq > 0 && result.ExecutionSeq <= auth.cooldownRecordedSeq
isDisabled := auth.Disabled || auth.Status == StatusDisabled
if result.Success {
if result.CompletedResponse && !result.SkipQuotaObservation && !isDisabled && !wasTerminalUnauthorized && !(auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now)) && !retainModelCooldown && !retainAuthCooldown && !retainCredentialRetry {
auth.InferenceVerified = true
auth.LastInferenceVerifiedAt = now
}
if isDisabled {
// A disabled credential (operator action or terminal TOS) stays
// unavailable until explicitly re-enabled; a late in-flight success
// must not reset model states or flip availability back to active.
auth.Unavailable = true
} else if wasTerminalUnauthorized {
if modelKey != "" {
state := ensureModelState(auth, modelKey)
modelState = state
resetModelState(state, now)
}
} else if auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now) {
// Retain active credential-scoped cooldown
} else if retainModelCooldown || retainAuthCooldown || retainCredentialRetry {
// Retain a cooldown recorded after this execution was dispatched.
} else if modelKey != "" {
state := ensureModelState(auth, modelKey)
modelState = state
resetModelState(state, now)
auth.credentialRetryAfter = time.Time{}
auth.nonAuthCredentialRetryAfter = time.Time{}
auth.nonAuthCredentialLastError = nil
updateAggregatedAvailability(auth, now)
if !auth.Disabled && auth.Status != StatusDisabled && !hasModelError(auth, now) && !effectiveAuthForceUntil(auth).After(now) {
auth.LastError = nil
auth.StatusMessage = ""
auth.Status = StatusActive
}
} else {
clearAuthStateOnSuccess(auth, now)
}
} else {
if modelKey != "" {
if auth.Disabled || auth.Status == StatusDisabled {
// A disabled credential (operator action or terminal TOS) is
// terminal until explicitly re-enabled; a late failure must not
// overwrite its disabled state or arm a new cooldown.
} else if !shouldSkipCredentialCooldown(result.Error) {
disableCooling := m.cooldownDisabledForAuth(auth)
if result.Error != nil && result.Error.Code == ErrorCodeForceCooldown {
disableCooling = false
}
state := ensureModelState(auth, modelKey)
modelState = state
// Normalize a naturally expired credential-quota wrap before
// applying a new result (issue #6690): reinstate the
// independent layer (or a clean slate) and drop stale wrap
// provenance so a future window snapshots fresh state.
if (state.credentialQuotaPropagated || state.Quota.Reason == "credential_quota") &&
!(auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now)) {
if shadow := state.credentialQuotaShadow; shadow != nil && isModelStateActiveCooldown(shadow, now) {
preservedSeq := shadow.cooldownRecordedSeq
*state = *shadow.cloneShallow()
state.cooldownRecordedSeq = preservedSeq
} else {
resetModelState(state, now)
}
}
// Two-layer handling (issue #6690): while an auth-level
// credential quota wraps the credential, a non-credential
// own failure is applied to the independent layer (shadow)
// so its deadline/backoff never inherit the propagated
// quota; the outer fields are then composed from both
// layers. Credential-scoped failures refresh the outer
// layer and are never independent.
outerState := state
savedOuterState := state.cloneShallow()
ownLayerPending := !result.CredentialScope &&
auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now)
if ownLayerPending {
if state.credentialQuotaShadow != nil {
preservedSeq := state.credentialQuotaShadow.cooldownRecordedSeq
*state = *state.credentialQuotaShadow.cloneShallow()
state.cooldownRecordedSeq = preservedSeq
} else {
fresh := &ModelState{}
*state = *fresh
}
}
seedInitialForceCooldownDeadlines(auth)
seedModelForceCooldownDeadline(state)
state.Unavailable = true
state.Status = StatusError
state.UpdatedAt = now
prevModelRetryAfter := state.NextRetryAfter
prevModelForceUntil := state.forceCooldownUntil
prevModelQuota := cooldownFieldsOf(state.Quota)
prevModelQuotaSignals := maps.Clone(state.Quota.cooldownSignals)
var ownModelForceDeadline time.Time
var ownModelNonQuotaDeadline time.Time
isForceResult := isForceCooldownError(result.Error)
appliedCredQuotaWrap := false
if result.Error != nil {
state.LastError = cloneError(result.Error)
state.StatusMessage = result.Error.Message
if !wasTerminalUnauthorized {
auth.LastError = cloneError(result.Error)
auth.StatusMessage = result.Error.Message
}
}
statusCode := statusCodeFromResult(result.Error)
if isModelSupportResultError(result.Error) {
if disableCooling {
state.NextRetryAfter = time.Time{}
} else if result.RetryAfter != nil && *result.RetryAfter > 0 {
state.NextRetryAfter = now.Add(*result.RetryAfter)
} else {
next := now.Add(12 * time.Hour)
state.NextRetryAfter = next
}
ownModelNonQuotaDeadline = state.NextRetryAfter
} else if isCloudflareChallengeResultError(result.Error) {
next, backoffLevel := nextCloudflareCooldown(state.Quota.BackoffLevel, disableCooling, now)
state.NextRetryAfter = next
ownModelNonQuotaDeadline = next
state.StatusMessage = "cloudflare challenge"
if auth.LastError != nil && !wasTerminalUnauthorized {
auth.StatusMessage = "cloudflare challenge"
}
applyCooldownFields(&state.Quota, QuotaState{
Exceeded: true,
Reason: "cloudflare challenge",
NextRecoverAt: next,
BackoffLevel: backoffLevel,
})
} else if isInvalidGrantResultError(result.Error) || isCreditBalanceTooLowResultError(result.Error) {
if disableCooling {
state.NextRetryAfter = time.Time{}
} else {
state.NextRetryAfter = now.Add(30 * time.Minute)
}
ownModelNonQuotaDeadline = state.NextRetryAfter
} else if !result.CredentialScope && isTransientRateLimitResultError(result.Error) && statusCode == http.StatusTooManyRequests {
// Transient per-turn capacity rejection (issue 7017): cool
// only the reporting model until the provider retry hint.
// The credential quota is not exhausted, so no quota state
// is written and the quota exponential backoff never moves;
// a still-live genuine quota deadline is left untouched and
// keeps blocking through the generic retry branches below.
next := transientRateLimitRetryAfter(now, result.RetryAfter, disableCooling)
ownModelNonQuotaDeadline = next
if isForceResult {
ownModelForceDeadline = next
}
if !next.IsZero() && state.Quota.Exceeded && state.Quota.NextRecoverAt.After(next) {
next = state.Quota.NextRecoverAt
}
if next.IsZero() && state.Quota.Exceeded && state.Quota.NextRecoverAt.After(now) {
next = state.Quota.NextRecoverAt
}
state.NextRetryAfter = next
state.Unavailable = !next.IsZero()
} else if isTOSViolationResultError(result.Error) {
// Terminal TOS suspension: keep the model state blocked
// without a retry deadline; the auth-level terminal state
// is finalized below.
state.NextRetryAfter = time.Time{}
state.Unavailable = true
state.Status = StatusDisabled
state.StatusMessage = "tos_violation"
} else {
switch statusCode {
case 401, 402, 403:
if disableCooling {
state.NextRetryAfter = time.Time{}
} else {
next := now.Add(30 * time.Minute)
state.NextRetryAfter = next
}
case 404:
if disableCooling {
state.NextRetryAfter = time.Time{}
} else if result.RetryAfter != nil && *result.RetryAfter > 0 {
state.NextRetryAfter = now.Add(*result.RetryAfter)
} else {
next := now.Add(12 * time.Hour)
state.NextRetryAfter = next
}
case 429:
prevStateQuotaActive := state.Quota.Exceeded && state.Quota.NextRecoverAt.After(now)
prevStateRecoverAt := state.Quota.NextRecoverAt
prevStateCooldownSignals := maps.Clone(state.Quota.cooldownSignals)
var next time.Time
var credentialNext time.Time
keptPriorStateQuota := false
backoffLevel := state.Quota.BackoffLevel
if result.CredentialScope {
backoffLevel = 0
if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" {
backoffLevel = auth.Quota.BackoffLevel
}
}
if !disableCooling {
reusedOpenWindow := false
var baseBackoffBefore int
if result.RetryAfter != nil {
cooldown := *result.RetryAfter
if cooldown < minQuotaCooldownFloor {
cooldown = minQuotaCooldownFloor
}
next = now.Add(cooldown).Round(0)
} else {
quotaForFailure := state.Quota
if result.CredentialScope {
if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" {
quotaForFailure = auth.Quota
} else {
quotaForFailure.NextRecoverAt = time.Time{}
quotaForFailure.BackoffLevel = 0
}
}
reusedOpenWindow = quotaForFailure.NextRecoverAt.After(now)
baseBackoffBefore = quotaForFailure.BackoffLevel
next, backoffLevel = quotaCooldownAfterFailure(quotaForFailure, now)
}
credentialNext = next
if isForceResult {
if result.RetryAfter != nil || !reusedOpenWindow {
ownModelForceDeadline = credentialNext
} else if result.CredentialScope {
if auth.credQuotaForceUntil.After(now) {
ownModelForceDeadline = auth.credQuotaForceUntil
} else {
ownModelForceDeadline, _ = quotaCooldownAfterFailure(QuotaState{BackoffLevel: baseBackoffBefore}, now)
}
} else if prevModelForceUntil.After(now) {
ownModelForceDeadline = prevModelForceUntil
} else {
ownModelForceDeadline, _ = quotaCooldownAfterFailure(QuotaState{BackoffLevel: baseBackoffBefore}, now)
}
}
if prevStateQuotaActive && (prevStateRecoverAt.After(next) || (!result.CredentialScope && result.RetryAfter == nil)) {
next = prevStateRecoverAt
keptPriorStateQuota = true
} else if result.CredentialScope && result.RetryAfter == nil && auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now) {
// quotaCooldownAfterFailure reused the open credential_quota window.
keptPriorStateQuota = prevStateQuotaActive && prevStateRecoverAt.Equal(next)
}
}
var cooldownSignals map[string]string
if !disableCooling {
if keptPriorStateQuota {
cooldownSignals = prevStateCooldownSignals
} else {
cooldownSignals = rejectedClaudeSignalsForCooldown(result.Provider, responseHeaders)
if len(cooldownSignals) == 0 && prevStateQuotaActive && prevStateRecoverAt.Equal(next) {
cooldownSignals = prevStateCooldownSignals
}
}
}
state.NextRetryAfter = next
applyCooldownFields(&state.Quota, QuotaState{
Exceeded: true,
Reason: "quota",
NextRecoverAt: next,
BackoffLevel: backoffLevel,
cooldownSignals: cooldownSignals,
})
// Non-credential own failures re-derive their independent layer
// below, so a stale shadow is dropped. A credential-scoped
// report must not erase an existing independent layer: it only
// refreshes the propagated outer layer.
if !result.CredentialScope {
state.credentialQuotaShadow = nil
} else if !disableCooling && !state.credentialQuotaPropagated && savedOuterState.Quota.Reason != "credential_quota" {
// First wrap of the reporting model: preserve any live prior
// own block (including an ordinary model-scoped 429, which
// quota restoration does not supersede). An outer
// credential_quota label instead proves a prior wrap whose
// state must never be snapshotted as independence.
priorIndependent := prevModelRetryAfter.After(now) && !isDevinQuotaExhaustionMessage(devinStateErrorText(savedOuterState))
if priorIndependent {
preserved := savedOuterState.cloneShallow()
preserved.NextRetryAfter = prevModelRetryAfter
preserved.Unavailable = true
preserved.Status = StatusError
state.credentialQuotaShadow = preserved
} else {
state.credentialQuotaShadow = nil
}
}
if result.CredentialScope && !disableCooling {
appliedCredQuotaWrap = true
prevAuthCredActive := auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now)
prevAuthCredRecoverAt := auth.Quota.NextRecoverAt
prevAuthCredSignals := maps.Clone(auth.Quota.cooldownSignals)
var credCooldownSignals map[string]string
if prevAuthCredActive && (prevAuthCredRecoverAt.After(credentialNext) || result.RetryAfter == nil) {
credCooldownSignals = prevAuthCredSignals
} else {
credCooldownSignals = rejectedClaudeSignalsForCooldown(result.Provider, responseHeaders)
if len(credCooldownSignals) == 0 && prevAuthCredActive && prevAuthCredRecoverAt.Equal(credentialNext) {
credCooldownSignals = prevAuthCredSignals
}
}
propagateCredentialQuotaToModelStatesLocked(auth, state, credentialNext, backoffLevel, credCooldownSignals, prevAuthCredActive, prevAuthCredRecoverAt, now)
if !wasTerminalUnauthorized {
auth.Unavailable = true
authNext := credentialNext
if prevAuthCredActive && prevAuthCredRecoverAt.After(authNext) {
authNext = prevAuthCredRecoverAt
}
auth.Quota.Exceeded = true
auth.Quota.Reason = "credential_quota"
auth.Quota.NextRecoverAt = authNext
auth.Quota.BackoffLevel = backoffLevel
auth.Quota.cooldownSignals = maps.Clone(credCooldownSignals)
auth.Quota.cooldownEvaluated = true
auth.NextRetryAfter = authNext
if authNext.After(state.Quota.NextRecoverAt) || (prevAuthCredActive && result.RetryAfter == nil) {
state.Quota.NextRecoverAt = authNext
state.Quota.cooldownSignals = maps.Clone(credCooldownSignals)
if state.NextRetryAfter.Before(authNext) {
state.NextRetryAfter = authNext
}
}
if isForceResult && ownModelForceDeadline.After(auth.credQuotaForceUntil) {
auth.credQuotaForceUntil = ownModelForceDeadline
}
auth.forceCooldownSeeded = true
// The reporting model is now part of the wrap too: mark it
// so a later credential report from another model cannot
// snapshot this credential-derived state as independent.
state.credentialQuotaPropagated = true
}
}
case 408, 500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526:
state.NextRetryAfter = recoverableFailureRetryAfterWithHint(now, result.RetryAfter, disableCooling)
state.Unavailable = !state.NextRetryAfter.IsZero()
default:
state.NextRetryAfter = recoverableFailureRetryAfter(now, disableCooling)
state.Unavailable = !state.NextRetryAfter.IsZero()
}
if !isQuotaCooldownResultError(result.Error) {
ownModelNonQuotaDeadline = state.NextRetryAfter
}
}
if disableCooling && !isTOSViolationResultError(result.Error) && state.NextRetryAfter.IsZero() && state.Quota.NextRecoverAt.IsZero() {
state.Unavailable = false
state.Quota.Exceeded = false
}
if isForceResult && state.NextRetryAfter.IsZero() {
state.NextRetryAfter = now.Add(transientErrorCooldown)
state.Unavailable = true
}
if isForceResult && !isQuotaCooldownResultError(result.Error) && state.NextRetryAfter.After(ownModelNonQuotaDeadline) {
ownModelNonQuotaDeadline = state.NextRetryAfter
}
if ownModelNonQuotaDeadline.After(state.nonQuotaCooldownUntil) {
state.nonQuotaCooldownUntil = ownModelNonQuotaDeadline
}
if !appliedCredQuotaWrap {
if result.CredentialScope && auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.credQuotaForceUntil.After(now) && auth.Quota.NextRecoverAt.After(now) {
lastErr := cloneError(state.LastError)
statusMsg := state.StatusMessage
*state = *savedOuterState
state.LastError = lastErr
state.StatusMessage = statusMsg
state.Unavailable = true
state.Status = StatusError
if state.NextRetryAfter.Before(auth.credQuotaForceUntil) {
state.NextRetryAfter = auth.credQuotaForceUntil
}
if ownModelNonQuotaDeadline.After(state.nonQuotaCooldownUntil) {
state.nonQuotaCooldownUntil = ownModelNonQuotaDeadline
}
state.UpdatedAt = now
} else {
if isForceResult {
if ownModelForceDeadline.IsZero() {
ownModelForceDeadline = state.NextRetryAfter
}
if ownModelForceDeadline.After(prevModelForceUntil) {
state.forceCooldownUntil = ownModelForceDeadline
} else {
state.forceCooldownUntil = prevModelForceUntil
}
} else {
state.forceCooldownUntil = prevModelForceUntil
}
state.forceCooldownSeeded = true
if isForceResult && isQuotaCooldownResultError(result.Error) && ownModelForceDeadline.After(state.quotaForceCooldownUntil) {
state.quotaForceCooldownUntil = ownModelForceDeadline
}
if state.forceCooldownUntil.After(now) {
if state.NextRetryAfter.Before(state.forceCooldownUntil) {
state.NextRetryAfter = state.forceCooldownUntil
}
state.Unavailable = true
state.Status = StatusError
if (!state.Quota.Exceeded || state.Quota.NextRecoverAt.IsZero()) && prevModelQuota.Exceeded && prevModelQuota.NextRecoverAt.After(now) {
restored := prevModelQuota
if disableCooling && restored.NextRecoverAt.After(state.forceCooldownUntil) {
restored.NextRecoverAt = state.forceCooldownUntil
}
restored.cooldownSignals = prevModelQuotaSignals
applyCooldownFields(&state.Quota, restored)
}
}
}
}
// A later failure only extends a still-live cooldown; it never
// shortens one. A deliberate zero write (disableCooling) still
// clears the deadline.
if !state.NextRetryAfter.IsZero() && prevModelRetryAfter.After(state.NextRetryAfter) && prevModelRetryAfter.After(now) {
state.NextRetryAfter = prevModelRetryAfter
}
// Capture the dispatch watermark before composing layers so the
// independent shadow carries this failure's stale-success
// protection (issue #6690).
cooldownSeq := m.currentExecutionSeq()
state.cooldownRecordedSeq = cooldownSeq
if ownLayerPending {
// Compose the outer presentation from the saved propagated
// layer and the independent layer just updated in place.
independent := state.cloneShallow()
outer := savedOuterState.cloneShallow()
if isModelStateActiveCooldown(independent, now) {
outer.credentialQuotaShadow = independent
} else {
outer.credentialQuotaShadow = nil
}
outer.cooldownRecordedSeq = cooldownSeq
// A composed outer state is wrapped by definition; stamp it so
// a later credential report cannot mistake it for unwrapped.
outer.credentialQuotaPropagated = true
outer.Unavailable = true
outer.Status = StatusError
outer.UpdatedAt = now
outer.LastError = cloneError(state.LastError)
outer.StatusMessage = state.StatusMessage
if state.NextRetryAfter.After(outer.NextRetryAfter) {
outer.NextRetryAfter = state.NextRetryAfter
}
if auth.Quota.NextRecoverAt.After(outer.NextRetryAfter) {
outer.NextRetryAfter = auth.Quota.NextRecoverAt
}
if !(outer.Quota.Exceeded && outer.Quota.Reason == "credential_quota") {
applyCooldownFields(&outer.Quota, QuotaState{
Exceeded: true,
Reason: "credential_quota",
NextRecoverAt: auth.Quota.NextRecoverAt,
BackoffLevel: auth.Quota.BackoffLevel,
cooldownSignals: auth.Quota.cooldownSignals,
})
}
*outerState = *outer
state = outerState
modelState = state
}
if result.CredentialScope && !disableCooling && statusCodeFromResult(result.Error) == 429 {
auth.cooldownRecordedSeq = cooldownSeq
for _, otherState := range auth.ModelStates {
if otherState != nil && otherState != state {
otherState.cooldownRecordedSeq = cooldownSeq
}
}
}
if result.CredentialScope && !disableCooling && !appliedCredQuotaWrap && !isTOSViolationResultError(result.Error) && !wasTerminalUnauthorized && ownModelNonQuotaDeadline.After(now) {
if ownModelNonQuotaDeadline.After(auth.credentialRetryAfter) {
auth.credentialRetryAfter = ownModelNonQuotaDeadline
}
if !isRefreshClearableCredentialError(result.Error) {
if ownModelNonQuotaDeadline.After(auth.nonAuthCredentialRetryAfter) {
auth.nonAuthCredentialRetryAfter = ownModelNonQuotaDeadline
auth.nonAuthCredentialLastError = cloneError(result.Error)
} else if auth.nonAuthCredentialLastError == nil {
auth.nonAuthCredentialLastError = cloneError(result.Error)
}
}
if ownModelNonQuotaDeadline.After(auth.nonQuotaCooldownUntil) {
auth.nonQuotaCooldownUntil = ownModelNonQuotaDeadline
}
if isForceResult {
if ownModelNonQuotaDeadline.After(auth.forceCooldownUntil) {
auth.forceCooldownUntil = ownModelNonQuotaDeadline
}
auth.forceCooldownSeeded = true
}
auth.cooldownRecordedSeq = cooldownSeq
}
if isTOSViolationResultError(result.Error) {
auth.StatusMessage = "tos_violation"
auth.Status = StatusDisabled
auth.Disabled = true
auth.Unavailable = true
auth.NextRetryAfter = time.Time{}
persistTOSAppealMetadata(auth, result.Error)
} else {
auth.Status = StatusError
}
updateAggregatedAvailability(auth, now)
}
} else {
disableCooling := m.cooldownDisabledForAuth(auth)
if result.Error != nil && result.Error.Code == ErrorCodeForceCooldown {
disableCooling = false
}
if auth.Disabled || auth.Status == StatusDisabled {
// A disabled credential (operator action or terminal TOS) is
// terminal until explicitly re-enabled; a late auth-level failure
// must not overwrite its disabled state or arm a new cooldown.
} else if !wasTerminalUnauthorized {
prevAuthCredActive := auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now)
prevAuthCredRecoverAt := auth.Quota.NextRecoverAt
prevAuthCredSignals := maps.Clone(auth.Quota.cooldownSignals)
isolatedModelAgg := result.CredentialScope && len(auth.ModelStates) > 0 && auth.Quota.Reason != "credential_quota"
var savedModelAggQuota QuotaState
if isolatedModelAgg {
savedModelAggQuota = auth.Quota.Clone()
authOwnRetry := auth.credentialRetryAfter
if activeAuthForce := effectiveAuthForceUntil(auth); activeAuthForce.After(authOwnRetry) {
authOwnRetry = activeAuthForce
}
auth.NextRetryAfter = authOwnRetry
auth.Quota = QuotaState{
ObservedAt: savedModelAggQuota.ObservedAt,
Signals: maps.Clone(savedModelAggQuota.Signals),
}
}
prevAuthQuotaActive := auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(now)
prevAuthRecoverAt := auth.Quota.NextRecoverAt
prevAuthCooldownSignals := maps.Clone(auth.Quota.cooldownSignals)
keptPriorAuthQuota := false
if !disableCooling && statusCodeFromResult(result.Error) == 429 && prevAuthQuotaActive {
if result.RetryAfter == nil {
keptPriorAuthQuota = true
} else {
cooldown := *result.RetryAfter
if cooldown < minQuotaCooldownFloor {
cooldown = minQuotaCooldownFloor
}
if prevAuthRecoverAt.After(now.Add(cooldown).Round(0)) {
keptPriorAuthQuota = true
}
}
}
ownAuthForceDeadline, ownAuthRetryDeadline := applyAuthFailureState(auth, result.Error, result.RetryAfter, now, disableCooling)
cooldownSeq := m.currentExecutionSeq()
isQuota429 := !disableCooling &&
statusCodeFromResult(result.Error) == 429 &&
!isTransientRateLimitResultError(result.Error) &&
!isCloudflareChallengeResultError(result.Error) &&
!isCreditBalanceTooLowResultError(result.Error) &&
!isInvalidGrantResultError(result.Error) &&
auth.Quota.Exceeded
if isQuota429 {
auth.Quota.cooldownEvaluated = true
if keptPriorAuthQuota {
auth.Quota.cooldownSignals = prevAuthCooldownSignals
} else if rejectedSignals := rejectedClaudeSignalsForCooldown(result.Provider, responseHeaders); len(rejectedSignals) > 0 {
auth.Quota.cooldownSignals = rejectedSignals
} else if prevAuthQuotaActive && auth.Quota.NextRecoverAt.Equal(prevAuthRecoverAt) {
auth.Quota.cooldownSignals = prevAuthCooldownSignals
} else {
auth.Quota.cooldownSignals = nil
}
if result.CredentialScope && !shouldSkipCredentialCooldown(result.Error) {
auth.Quota.Reason = "credential_quota"
if isForceCooldownError(result.Error) && ownAuthForceDeadline.After(auth.credQuotaForceUntil) {
auth.credQuotaForceUntil = ownAuthForceDeadline
auth.forceCooldownSeeded = true
}
credSignals := auth.Quota.cooldownSignals
if prevAuthCredActive && (prevAuthCredRecoverAt.After(auth.Quota.NextRecoverAt) || result.RetryAfter == nil) {
credSignals = prevAuthCredSignals
}
propagateCredentialQuotaToModelStatesLocked(auth, nil, auth.Quota.NextRecoverAt, auth.Quota.BackoffLevel, credSignals, prevAuthCredActive, prevAuthCredRecoverAt, now)
for _, otherState := range auth.ModelStates {
if otherState != nil {
otherState.cooldownRecordedSeq = cooldownSeq
}
}
}
} else {
if result.CredentialScope && !disableCooling && !shouldSkipCredentialCooldown(result.Error) && !ownAuthRetryDeadline.IsZero() && ownAuthRetryDeadline.After(now) {
if ownAuthRetryDeadline.After(auth.credentialRetryAfter) {
auth.credentialRetryAfter = ownAuthRetryDeadline
}
if !isRefreshClearableCredentialError(result.Error) {
if ownAuthRetryDeadline.After(auth.nonAuthCredentialRetryAfter) {
auth.nonAuthCredentialRetryAfter = ownAuthRetryDeadline
auth.nonAuthCredentialLastError = cloneError(result.Error)
} else if auth.nonAuthCredentialLastError == nil {
auth.nonAuthCredentialLastError = cloneError(result.Error)
}
}
}
if isolatedModelAgg && !isCloudflareChallengeResultError(result.Error) && !isTOSViolationResultError(result.Error) {
savedModelAggQuota.Signals = mergeQuotaObservation(auth.Quota, savedModelAggQuota).Signals
applyCooldownFields(&auth.Quota, savedModelAggQuota)
updateAggregatedAvailability(auth, now)
}
}
auth.cooldownRecordedSeq = cooldownSeq
}
}
}
if rejectsToken {
tokenToReject := result.AccessToken
if tokenToReject == "" {
tokenToReject = authAccessTokenForRequest(ctx, auth, result.Options.Headers)
}
_ = recordRejectedTokenOnAuth(auth, tokenToReject)
}
if wasTerminalUnauthorized {
auth.Unavailable = true
auth.Status = StatusError
auth.NextRefreshAfter = time.Time{}
auth.NextRetryAfter = time.Time{}
} else if auth.Disabled || auth.Status == StatusDisabled {
// A terminal TOS_VIOLATION already deactivated the credential inside
// applyAuthFailureState; preserve that state instead of overwriting
// it with a generic error status.
} else if rejectsToken && authHasRefreshCredential(auth) {
if auth.NextRefreshAfter.IsZero() {
if !auth.LastRefreshedAt.IsZero() && now.Sub(auth.LastRefreshedAt) < refreshFailureBackoff {
auth.NextRefreshAfter = now.Add(refreshFailureBackoff)
} else {
auth.NextRefreshAfter = now
}
}
scheduleUnauthorizedRefresh = true
}
auth.Generation++
auth.UpdatedAt = now
if !result.SkipQuotaObservation {
auth.Quota.ObserveResponseHeadersForProvider(result.Provider, responseHeaders, now)
if modelState != nil {
modelState.Quota.ObserveResponseHeadersForProvider(result.Provider, responseHeaders, now)
}
}
_ = m.persistLocked(ctx, auth)
authSnapshot = auth.Clone()
if trackCooldownState {
cooldownRecordsAfter := m.cooldownStateRecordsForAuthLocked(auth, now)
cooldownStateChanged = !cooldownStateRecordsEqual(cooldownRecordsBefore, cooldownRecordsAfter)
}
}
m.mu.Unlock()
releaseMutation()
if m.scheduler != nil && authSnapshot != nil {
credentialScopedResult := result.CredentialScope || scheduleUnauthorizedRefresh
var targetModels []string
if !credentialScopedResult && modelKey != "" {
targetModels = append(targetModels, modelKey)
if routeKey := canonicalModelKey(result.RouteModel); routeKey != "" && !sameCanonicalModelKey(routeKey, modelKey) {
targetModels = append(targetModels, routeKey)
}
}
m.scheduler.upsertAuthResult(authSnapshot, targetModels, credentialScopedResult)
}
if authSnapshot != nil && cooldownStateChanged {
m.persistCooldownStates(context.Background())
}
// A newly exhausted Devin credential must join the refresh schedule so
// quota revalidation can detect early restoration (issue #6690).
if authSnapshot != nil {
if _, due := devinQuotaRevalidateAt(authSnapshot, now); due {
m.queueRefreshReschedule(result.AuthID)
} else if scheduleUnauthorizedRefresh {
m.queueRefreshReschedule(result.AuthID)
}
}
supportedModels, regEpoch := registry.GetGlobalRegistry().GetModelsAndEpochForClient(result.AuthID)
reg := registry.GetGlobalRegistry()
projections := make([]registry.ClientModelProjection, 0, len(supportedModels))
for _, sm := range supportedModels {
if sm == nil || strings.TrimSpace(sm.ID) == "" {
continue
}
projections = append(projections, m.clientModelProjectionForAuth(authSnapshot, sm.ID, now))
}
if authSnapshot != nil && len(projections) > 0 {
reg.ApplyClientModelProjections(result.AuthID, regEpoch, authSnapshot.Generation, projections)
}
m.hook.OnResult(ctx, result)
m.publishErrorEvent(result, authSnapshot)
m.updateSessionAffinity(result)
}
func (m *Manager) updateSessionAffinity(result Result) {
if m == nil {
return
}
sel := m.Selector()
if affinity, ok := sel.(interface {
OnResult(Result)
}); ok && affinity != nil {
affinity.OnResult(result)
}
}
func (m *Manager) recordExecutionResult(ctx context.Context, result Result, auth *Auth, ephemeral bool) {
if auth != nil {
if result.CredentialVersion == 0 {
result.CredentialVersion = auth.CredentialVersion
}
if result.RegistrationEpoch == 0 {
result.RegistrationEpoch = auth.RegistrationEpoch
}
if result.AccessToken == "" {
result.AccessToken = authAccessTokenForRequest(ctx, auth, result.Options.Headers)
}
if result.RefreshToken == "" {
result.RefreshToken = authRefreshToken(auth)
}
}
if !ephemeral {
m.MarkResult(ctx, result)
return
}
m.reportHomeResult(ctx, result, auth)
}
// reportHomeResult only observes a Home dispatch result and never updates local auth state.
func (m *Manager) reportHomeResult(ctx context.Context, result Result, auth *Auth) {
if m == nil || result.AuthID == "" {
return
}
var snapshot *Auth
if auth != nil {
snapshot = auth.Clone()
}
m.hook.OnResult(ctx, result)
m.publishErrorEvent(result, snapshot)
m.updateSessionAffinity(result)
}
func (m *Manager) recordAvailabilityNeutralResult(ctx context.Context, result Result) {
if result.AuthID == "" {
return
}
var authSnapshot *Auth
releaseMutation := m.lockAuthMutation(result.AuthID)
defer releaseMutation()
m.mu.Lock()
if auth, ok := m.auths[result.AuthID]; ok && auth != nil {
if isStaleExecutionResult(result, auth) {
m.mu.Unlock()
releaseMutation()
m.hook.OnResult(ctx, result)
m.publishErrorEvent(result, nil)
return
}
now := time.Now()
auth.recordRecentRequest(now, result.Success)
if result.Success {
auth.Success++
} else {
auth.Failed++
}
auth.Generation++
auth.UpdatedAt = now
_ = m.persistLocked(ctx, auth)
authSnapshot = auth.Clone()
}
m.mu.Unlock()
releaseMutation()
m.hook.OnResult(ctx, result)
m.publishErrorEvent(result, authSnapshot)
}
func modelStateKey(auth *Auth, model string) string {
if isCopilotAuth(auth) {
// Keep restored route IDs intact until the Copilot catalog can resolve them.
return strings.TrimSpace(model)
}
return canonicalModelKey(model)
}
func modelStateLookupKey(auth *Auth, model string) string {
return strings.ToLower(modelStateKey(auth, model))
}
func existingModelState(auth *Auth, model string) *ModelState {
model = canonicalModelKey(model)
if auth == nil || model == "" || len(auth.ModelStates) == 0 {
return nil
}
if state, ok := auth.ModelStates[model]; ok && state != nil {
return state
}
lookupKey := canonicalModelLookupKey(model)
if lookupKey == "" {
return nil
}
var matched *ModelState
for stateModel, state := range auth.ModelStates {
if state == nil || modelStateLookupKey(auth, stateModel) != lookupKey {
continue
}
if matched == nil {
matched = state
} else {
matched = mergeModelState(matched.Clone(), state)
}
}
return matched
}
func ensureModelState(auth *Auth, model string) *ModelState {
model = modelStateKey(auth, model)
if auth == nil || model == "" {
return nil
}
normalizeModelStates(auth)
if auth.ModelStates == nil {
auth.ModelStates = make(map[string]*ModelState)
}
if state, ok := auth.ModelStates[model]; ok && state != nil {
return state
}
lookupKey := modelStateLookupKey(auth, model)
for stateModel, state := range auth.ModelStates {
if modelStateLookupKey(auth, stateModel) == lookupKey && state != nil {
return state
}
}
state := &ModelState{Status: StatusActive}
auth.ModelStates[model] = state
return state
}
func normalizeModelStates(auth *Auth) bool {
if auth == nil || len(auth.ModelStates) == 0 {
return false
}
keys := make([]string, 0, len(auth.ModelStates))
for model := range auth.ModelStates {
keys = append(keys, model)
}
if len(keys) > 1 {
sort.Strings(keys)
}
normalized := make(map[string]*ModelState, len(auth.ModelStates))
keyByLookup := make(map[string]string, len(auth.ModelStates))
changed := false
for _, model := range keys {
state := auth.ModelStates[model]
modelKey := modelStateKey(auth, model)
if modelKey == "" {
modelKey = strings.TrimSpace(model)
}
if modelKey != model {
changed = true
}
lookupKey := modelStateLookupKey(auth, modelKey)
if lookupKey == "" {
lookupKey = strings.ToLower(modelKey)
}
if existingKey, ok := keyByLookup[lookupKey]; ok {
var existingClone *ModelState
if normalized[existingKey] != nil {
existingClone = normalized[existingKey].Clone()
}
merged := mergeModelState(existingClone, state)
targetKey := existingKey
if modelKey == lookupKey && existingKey != lookupKey {
delete(normalized, existingKey)
targetKey = modelKey
keyByLookup[lookupKey] = targetKey
}
normalized[targetKey] = merged
changed = true
continue
}
keyByLookup[lookupKey] = modelKey
normalized[modelKey] = state
}
if changed {
auth.ModelStates = normalized
}
return changed
}
func mergeModelState(target, source *ModelState) *ModelState {
if target == nil {
return source
}
if source == nil {
return target
}
preferred := target
fallback := source
if source.UpdatedAt.After(target.UpdatedAt) {
preferred = source
fallback = target
}
merged := ModelState{
Status: preferred.Status,
StatusMessage: preferred.StatusMessage,
Unavailable: target.Unavailable || source.Unavailable,
NextRetryAfter: target.NextRetryAfter,
LastError: cloneError(preferred.LastError),
Quota: QuotaState{
Exceeded: target.Quota.Exceeded || source.Quota.Exceeded,
Reason: preferred.Quota.Reason,
NextRecoverAt: target.Quota.NextRecoverAt,
BackoffLevel: target.Quota.BackoffLevel,
cooldownEvaluated: target.Quota.cooldownEvaluated || source.Quota.cooldownEvaluated,
},
UpdatedAt: target.UpdatedAt,
}
merged.Quota = mergeQuotaObservation(merged.Quota, fallback.Quota)
merged.Quota = mergeQuotaObservation(merged.Quota, preferred.Quota)
if source.Quota.NextRecoverAt.After(target.Quota.NextRecoverAt) {
merged.Quota.cooldownSignals = maps.Clone(source.Quota.cooldownSignals)
} else if target.Quota.NextRecoverAt.After(source.Quota.NextRecoverAt) {
merged.Quota.cooldownSignals = maps.Clone(target.Quota.cooldownSignals)
} else if len(preferred.Quota.cooldownSignals) > 0 {
merged.Quota.cooldownSignals = maps.Clone(preferred.Quota.cooldownSignals)
} else if len(fallback.Quota.cooldownSignals) > 0 {
merged.Quota.cooldownSignals = maps.Clone(fallback.Quota.cooldownSignals)
}
if source.NextRetryAfter.After(merged.NextRetryAfter) {
merged.NextRetryAfter = source.NextRetryAfter
}
if source.Quota.NextRecoverAt.After(merged.Quota.NextRecoverAt) {
merged.Quota.NextRecoverAt = source.Quota.NextRecoverAt
}
if source.Quota.BackoffLevel > merged.Quota.BackoffLevel {
merged.Quota.BackoffLevel = source.Quota.BackoffLevel
}
if source.UpdatedAt.After(merged.UpdatedAt) {
merged.UpdatedAt = source.UpdatedAt
}
// Keep the strongest cooldown watermark from either side so merging a
// restored or alias-migrated state into a live auth cannot reopen the
// stale-success race. Both sides stay zero across a process restart,
// where restored cooldowns must remain clearable by any success.
merged.cooldownRecordedSeq = target.cooldownRecordedSeq
if source.cooldownRecordedSeq > merged.cooldownRecordedSeq {
merged.cooldownRecordedSeq = source.cooldownRecordedSeq
}
merged.forceCooldownUntil = target.forceCooldownUntil
if source.forceCooldownUntil.After(merged.forceCooldownUntil) {
merged.forceCooldownUntil = source.forceCooldownUntil
}
merged.quotaForceCooldownUntil = target.quotaForceCooldownUntil
if source.quotaForceCooldownUntil.After(merged.quotaForceCooldownUntil) {
merged.quotaForceCooldownUntil = source.quotaForceCooldownUntil
}
merged.nonQuotaCooldownUntil = target.nonQuotaCooldownUntil
if source.nonQuotaCooldownUntil.After(merged.nonQuotaCooldownUntil) {
merged.nonQuotaCooldownUntil = source.nonQuotaCooldownUntil
}
merged.forceCooldownSeeded = target.forceCooldownSeeded || source.forceCooldownSeeded
if merged.StatusMessage == "" {
merged.StatusMessage = fallback.StatusMessage
}
if merged.LastError == nil {
merged.LastError = cloneError(fallback.LastError)
}
if merged.Quota.Reason == "" {
merged.Quota.Reason = fallback.Quota.Reason
}
if target.Status == StatusDisabled || source.Status == StatusDisabled {
merged.Status = StatusDisabled
} else if merged.Unavailable || merged.Quota.Exceeded {
merged.Status = StatusError
}
// Keep the independent layer and wrap provenance from either side
// (newer wins) while the merged state is wrapped. The reporting model
// carries outer reason "quota", so wrap detection must include the
// marker and shadow presence, not only the propagated label.
if merged.Quota.Reason == "credential_quota" || merged.Quota.Reason == "quota" ||
target.credentialQuotaPropagated || source.credentialQuotaPropagated ||
target.credentialQuotaShadow != nil || source.credentialQuotaShadow != nil {
if target.credentialQuotaShadow != nil && source.credentialQuotaShadow != nil {
merged.credentialQuotaShadow = mergeModelState(target.credentialQuotaShadow.cloneShallow(), source.credentialQuotaShadow.cloneShallow())
} else if preferred.credentialQuotaShadow != nil {
merged.credentialQuotaShadow = preferred.credentialQuotaShadow.cloneShallow()
} else if fallback.credentialQuotaShadow != nil {
merged.credentialQuotaShadow = fallback.credentialQuotaShadow.cloneShallow()
}
merged.credentialQuotaPropagated = target.credentialQuotaPropagated || source.credentialQuotaPropagated
}
*target = merged
return target
}
func resetModelState(state *ModelState, now time.Time) {
if state == nil {
return
}
state.Unavailable = false
state.Status = StatusActive
state.StatusMessage = ""
state.NextRetryAfter = time.Time{}
state.LastError = nil
applyCooldownFields(&state.Quota, QuotaState{})
state.cooldownRecordedSeq = 0
state.forceCooldownUntil = time.Time{}
state.quotaForceCooldownUntil = time.Time{}
state.nonQuotaCooldownUntil = time.Time{}
state.credentialQuotaShadow = nil
state.credentialQuotaPropagated = false
state.UpdatedAt = now
}
func isModelStateActiveCooldown(state *ModelState, now time.Time) bool {
if state == nil {
return false
}
if state.Status == StatusDisabled {
return true
}
if !state.NextRetryAfter.IsZero() && state.NextRetryAfter.After(now) {
return true
}
if !state.Quota.NextRecoverAt.IsZero() && state.Quota.NextRecoverAt.After(now) {
return true
}
if state.Quota.Exceeded && state.Quota.NextRecoverAt.IsZero() {
return true
}
return false
}
func (m *Manager) registryModelsForAuthAndModel(auth *Auth, authID, modelKey string) []string {
authID = strings.TrimSpace(authID)
modelKey = strings.TrimSpace(modelKey)
if authID == "" && auth != nil {
authID = auth.ID
}
if auth == nil && m != nil && authID != "" {
m.mu.RLock()
auth = m.auths[authID]
m.mu.RUnlock()
}
if authID == "" || modelKey == "" {
return nil
}
targetCanonical := canonicalModelLookupKey(modelKey)
supportedModels := registry.GetGlobalRegistry().GetModelsForClient(authID)
if len(supportedModels) == 0 {
return nil
}
out := make([]string, 0, len(supportedModels))
seen := make(map[string]struct{}, len(supportedModels))
for _, model := range supportedModels {
if model == nil || strings.TrimSpace(model.ID) == "" {
continue
}
regModelID := strings.TrimSpace(model.ID)
stateKey := ""
if m != nil {
stateKey = m.selectionModelKeyForAuth(auth, regModelID)
}
if stateKey == "" {
stateKey = canonicalModelKey(regModelID)
}
if canonicalModelLookupKey(stateKey) == targetCanonical {
if _, exists := seen[regModelID]; !exists {
seen[regModelID] = struct{}{}
out = append(out, regModelID)
}
}
}
if len(out) == 0 {
return nil
}
return out
}
func modelStateIsClean(state *ModelState) bool {
if state == nil {
return true
}
if state.Status != StatusActive {
return false
}
if state.Unavailable || state.StatusMessage != "" || !state.NextRetryAfter.IsZero() || state.LastError != nil {
return false
}
if state.Quota.Exceeded || state.Quota.Reason != "" || !state.Quota.NextRecoverAt.IsZero() || state.Quota.BackoffLevel != 0 {
return false
}
return true
}
func updateAggregatedAvailability(auth *Auth, now time.Time) {
if auth == nil {
return
}
// A disabled credential (operator action or terminal TOS suspension) stays
// unavailable regardless of sibling model states.
if auth.Disabled || auth.Status == StatusDisabled {
auth.Unavailable = true
return
}
// A terminal unauthorized credential stays blocked until its tokens change.
// Model-level results must not make it selectable again.
if hasUnauthorizedAuthFailure(auth) {
auth.Unavailable = true
return
}
if !auth.credentialRetryAfter.IsZero() && !auth.credentialRetryAfter.After(now) {
auth.credentialRetryAfter = time.Time{}
}
if !auth.nonAuthCredentialRetryAfter.IsZero() && !auth.nonAuthCredentialRetryAfter.After(now) {
auth.nonAuthCredentialRetryAfter = time.Time{}
auth.nonAuthCredentialLastError = nil
}
if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now) {
auth.Unavailable = true
if auth.NextRetryAfter.Before(auth.Quota.NextRecoverAt) {
auth.NextRetryAfter = auth.Quota.NextRecoverAt
}
if activeAuthForce := effectiveAuthForceUntil(auth); activeAuthForce.After(now) && auth.NextRetryAfter.Before(activeAuthForce) {
auth.NextRetryAfter = activeAuthForce
}
if auth.credentialRetryAfter.After(now) && auth.NextRetryAfter.Before(auth.credentialRetryAfter) {
auth.NextRetryAfter = auth.credentialRetryAfter
}
return
}
activeAuthForce := effectiveAuthForceUntil(auth)
if !activeAuthForce.After(now) && !auth.forceCooldownUntil.IsZero() {
auth.forceCooldownUntil = time.Time{}
}
if len(auth.ModelStates) == 0 {
if activeAuthForce.After(now) {
auth.Unavailable = true
if auth.NextRetryAfter.Before(activeAuthForce) {
auth.NextRetryAfter = activeAuthForce
}
return
}
if auth.credentialRetryAfter.After(now) {
auth.Unavailable = true
if auth.NextRetryAfter.Before(auth.credentialRetryAfter) {
auth.NextRetryAfter = auth.credentialRetryAfter
}
return
}
clearAggregatedAvailability(auth)
return
}
allUnavailable := true
earliestRetry := time.Time{}
quotaExceeded := false
quotaRecover := time.Time{}
maxBackoffLevel := 0
hasState := false
for _, state := range auth.ModelStates {
if state == nil {
continue
}
hasState = true
stateUnavailable := false
if state.Status == StatusDisabled {
stateUnavailable = true
} else if state.Unavailable {
if state.NextRetryAfter.IsZero() {
stateUnavailable = false
} else if state.NextRetryAfter.After(now) {
stateUnavailable = true
if earliestRetry.IsZero() || state.NextRetryAfter.Before(earliestRetry) {
earliestRetry = state.NextRetryAfter
}
} else {
state.Unavailable = false
state.NextRetryAfter = time.Time{}
}
}
if !stateUnavailable {
allUnavailable = false
}
if state.Quota.Exceeded {
quotaExceeded = true
if quotaRecover.IsZero() || (!state.Quota.NextRecoverAt.IsZero() && state.Quota.NextRecoverAt.Before(quotaRecover)) {
quotaRecover = state.Quota.NextRecoverAt
}
if state.Quota.BackoffLevel > maxBackoffLevel {
maxBackoffLevel = state.Quota.BackoffLevel
}
}
}
if !hasState {
if activeAuthForce.After(now) {
auth.Unavailable = true
if auth.NextRetryAfter.Before(activeAuthForce) {
auth.NextRetryAfter = activeAuthForce
}
return
}
if auth.credentialRetryAfter.After(now) {
auth.Unavailable = true
if auth.NextRetryAfter.Before(auth.credentialRetryAfter) {
auth.NextRetryAfter = auth.credentialRetryAfter
}
return
}
clearAggregatedAvailability(auth)
return
}
auth.Unavailable = allUnavailable
if allUnavailable {
auth.NextRetryAfter = earliestRetry
} else {
auth.NextRetryAfter = time.Time{}
}
if activeAuthForce.After(now) {
auth.Unavailable = true
if auth.NextRetryAfter.Before(activeAuthForce) {
auth.NextRetryAfter = activeAuthForce
}
}
if auth.credentialRetryAfter.After(now) {
auth.Unavailable = true
if auth.NextRetryAfter.Before(auth.credentialRetryAfter) {
auth.NextRetryAfter = auth.credentialRetryAfter
}
}
if quotaExceeded {
auth.Quota.Exceeded = true
auth.Quota.Reason = "quota"
if auth.Quota.NextRecoverAt.After(quotaRecover) {
quotaRecover = auth.Quota.NextRecoverAt
}
auth.Quota.NextRecoverAt = quotaRecover
auth.Quota.BackoffLevel = maxBackoffLevel
} else if auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(now) {
// Retain active auth-level quota cooldown
} else {
applyCooldownFields(&auth.Quota, QuotaState{})
}
}
func clearAggregatedAvailability(auth *Auth) {
if auth == nil {
return
}
auth.Unavailable = false
auth.NextRetryAfter = time.Time{}
applyCooldownFields(&auth.Quota, QuotaState{})
}
func hasModelError(auth *Auth, now time.Time) bool {
if auth == nil || len(auth.ModelStates) == 0 {
return false
}
for _, state := range auth.ModelStates {
if state == nil {
continue
}
if state.LastError != nil {
return true
}
if state.Status == StatusError {
if state.Unavailable && (state.NextRetryAfter.IsZero() || state.NextRetryAfter.After(now)) {
return true
}
}
}
return false
}
func propagateCredentialQuotaToModelStatesLocked(auth *Auth, exclude *ModelState, credentialNext time.Time, backoffLevel int, credCooldownSignals map[string]string, prevAuthCredActive bool, prevAuthCredRecoverAt time.Time, now time.Time) {
if auth == nil {
return
}
for _, otherState := range auth.ModelStates {
if otherState == nil || otherState == exclude {
continue
}
// Normalize a naturally expired wrap first (issue #6690) so a
// stale marker cannot suppress snapshotting a fresh block.
if (otherState.credentialQuotaPropagated || otherState.Quota.Reason == "credential_quota") &&
!prevAuthCredActive {
if shadow := otherState.credentialQuotaShadow; shadow != nil && isModelStateActiveCooldown(shadow, now) {
preservedSeq := shadow.cooldownRecordedSeq
*otherState = *shadow.cloneShallow()
otherState.cooldownRecordedSeq = preservedSeq
} else {
resetModelState(otherState, now)
}
}
// On first wrap, preserve the sibling's independent layer
// beneath the propagated credential quota (issue #6690):
// a live own block is snapshotted; healthy/propagated-only
// states get none. The marker keeps repeated propagation
// from mistaking its own earlier layer for independence.
alreadyPropagated := otherState.credentialQuotaPropagated ||
otherState.Quota.Reason == "credential_quota"
if !alreadyPropagated {
if isModelStateActiveCooldown(otherState, now) && !isDevinQuotaExhaustionMessage(devinStateErrorText(otherState)) {
otherState.credentialQuotaShadow = otherState.cloneShallow()
} else {
otherState.credentialQuotaShadow = nil
}
}
otherState.credentialQuotaPropagated = true
otherState.Unavailable = true
otherState.Status = StatusError
otherQuotaNext := credentialNext
otherSignals := credCooldownSignals
if prevAuthCredActive && prevAuthCredRecoverAt.After(otherQuotaNext) {
otherQuotaNext = prevAuthCredRecoverAt
}
if otherState.Quota.Exceeded && otherState.Quota.NextRecoverAt.After(otherQuotaNext) {
otherQuotaNext = otherState.Quota.NextRecoverAt
otherSignals = maps.Clone(otherState.Quota.cooldownSignals)
}
otherRetryAfter := otherQuotaNext
// Propagation only extends a sibling's still-live
// per-model deadline; it never shortens one.
if !otherState.NextRetryAfter.IsZero() && otherState.NextRetryAfter.After(otherRetryAfter) {
otherRetryAfter = otherState.NextRetryAfter
}
otherState.NextRetryAfter = otherRetryAfter
applyCooldownFields(&otherState.Quota, QuotaState{
Exceeded: true,
Reason: "credential_quota",
NextRecoverAt: otherQuotaNext,
BackoffLevel: backoffLevel,
cooldownSignals: otherSignals,
})
}
}
func clearAuthStateOnSuccess(auth *Auth, now time.Time) {
if auth == nil {
return
}
if auth.Disabled || auth.Status == StatusDisabled {
auth.Unavailable = true
return
}
if hasUnauthorizedAuthFailure(auth) {
auth.Unavailable = true
return
}
for _, state := range auth.ModelStates {
if state == nil {
continue
}
if state.credentialQuotaPropagated || state.Quota.Reason == "credential_quota" {
if shadow := state.credentialQuotaShadow; shadow != nil && isModelStateActiveCooldown(shadow, now) {
preservedSeq := shadow.cooldownRecordedSeq
*state = *shadow.cloneShallow()
state.cooldownRecordedSeq = preservedSeq
} else {
resetModelState(state, now)
}
}
}
auth.Unavailable = false
auth.Status = StatusActive
auth.StatusMessage = ""
applyCooldownFields(&auth.Quota, QuotaState{})
auth.LastError = nil
auth.NextRetryAfter = time.Time{}
auth.cooldownRecordedSeq = 0
auth.forceCooldownUntil = time.Time{}
auth.quotaForceCooldownUntil = time.Time{}
auth.credQuotaForceUntil = time.Time{}
auth.credentialRetryAfter = time.Time{}
auth.nonAuthCredentialRetryAfter = time.Time{}
auth.nonAuthCredentialLastError = nil
auth.nonQuotaCooldownUntil = time.Time{}
auth.UpdatedAt = now
}
// isAuthActiveCooldown reports whether the auth carries a still-live
// auth-level cooldown recorded without a model key.
func isAuthActiveCooldown(auth *Auth, now time.Time) bool {
if auth == nil || auth.Disabled || auth.Status == StatusDisabled {
return false
}
if !auth.NextRetryAfter.IsZero() && auth.NextRetryAfter.After(now) {
return true
}
if !auth.Quota.NextRecoverAt.IsZero() && auth.Quota.NextRecoverAt.After(now) {
return true
}
if auth.Quota.Exceeded && auth.Quota.NextRecoverAt.IsZero() {
return true
}
return false
}
func cloneError(err *Error) *Error {
if err == nil {
return nil
}
return err.Clone()
}
func errorString(err error) string {
if err == nil {
return ""
}
return err.Error()
}
func statusCodeFromError(err error) int {
if err == nil {
return 0
}
type statusCoder interface {
StatusCode() int
}
var sc statusCoder
if errors.As(err, &sc) && sc != nil {
return sc.StatusCode()
}
return 0
}
func errorCodeFromError(err error) string {
if err == nil {
return ""
}
var authErr *Error
if errors.As(err, &authErr) && authErr != nil && strings.TrimSpace(authErr.Code) != "" {
return strings.TrimSpace(authErr.Code)
}
type errorCoder interface {
ErrorCode() string
}
var ec errorCoder
if errors.As(err, &ec) && ec != nil {
return strings.TrimSpace(ec.ErrorCode())
}
return ""
}
func isRequestScopedError(err error) bool {
if err == nil || isCredentialScopedError(err) {
return false
}
// A preserved terminal response belongs to this request unless the executor
// explicitly classifies the rejection as safe to retry on another credential.
if isRequestTerminatedError(err) {
return true
}
requestErr, ok := errors.AsType[cliproxyexecutor.RequestScopedError](err)
return ok && requestErr != nil && requestErr.IsRequestScoped()
}
// isStreamTruncatedError reports locally synthesized failures for accepted
// upstream streams that ended without a terminal frame. Unlike a received
// HTTP refusal, the generation was already submitted upstream.
func isStreamTruncatedError(err error) bool {
if err == nil || isCredentialScopedError(err) {
return false
}
truncatedErr, ok := errors.AsType[cliproxyexecutor.StreamTruncatedError](err)
if ok && truncatedErr != nil && truncatedErr.IsStreamTruncated() {
return true
}
var authErr *Error
if errors.As(err, &authErr) && authErr != nil && authErr.Code == streamTruncatedErrorCode {
return true
}
return false
}
func isStreamTruncatedResultError(err *Error) bool {
return err != nil && err.Code == streamTruncatedErrorCode
}
// ResultErrorFromError classifies an execution or transport error into an auth.Error
// while preserving lifecycle and transient transport classifications so MarkResult
// does not cool credentials on client cancellations or network drops.
func ResultErrorFromError(err error) *Error {
return resultErrorFromError(err)
}
func resultErrorFromError(err error) *Error {
if err == nil {
return nil
}
var sourceErr *Error
var resultErr *Error
if errors.As(err, &sourceErr) && sourceErr != nil {
resultErr = cloneError(sourceErr)
} else {
resultErr = &Error{Message: err.Error()}
}
if resultErr.Code == "" {
resultErr.Code = errorCodeFromError(err)
}
if resultErr.HTTPStatus == 0 {
resultErr.HTTPStatus = statusCodeFromError(err)
}
type tosMetadataProvider interface {
TOSAppealMetadata() map[string]string
}
var tmp tosMetadataProvider
if errors.As(err, &tmp) && tmp != nil {
if meta := tmp.TOSAppealMetadata(); len(meta) > 0 {
resultErr.Metadata = meta
}
}
switch {
case isCredentialScopedError(err):
// A credential-scoped failure always stays on the credential quota path;
// normalize conflicting model-only or cooldown-skipping codes so MarkResult
// cannot downgrade whole-credential exhaustion.
switch strings.ToLower(strings.TrimSpace(resultErr.Code)) {
case transientRateLimitErrorCode,
requestScopedErrorCode,
streamTruncatedErrorCode,
connectionLifecycleErrorCode,
transientTransportErrorCode:
resultErr.Code = credentialQuotaErrorCode
}
case isStreamTruncatedError(err):
// Preserve the local truncation classification alongside the public
// 502 status so MarkResult can skip credential cooldown.
resultErr.Code = streamTruncatedErrorCode
case isTransientRateLimitError(err):
// Preserve the transient capacity classification alongside the public
// 429 status so MarkResult cools only the reporting model until the
// provider retry hint instead of recording quota exhaustion.
resultErr.Code = transientRateLimitErrorCode
case isExplicitModelNotFoundError(err, ""):
if resultErr.Code == "" || resultErr.Code == requestScopedErrorCode {
resultErr.Code = "model_not_found"
}
case isRequestScopedError(err) || isRequestInvalidError(err):
// Prefer true request-scoped faults (including Claude OAuth cancellation)
// over the broader connection-lifecycle classification.
resultErr.Code = requestScopedErrorCode
case isConfiguredTLSHandshakeTimeoutError(err):
// An internal TLS establishment deadline is a transient transport
// failure, not a caller-owned context lifecycle event.
if resultErr.Code == "" || resultErr.Code == transientTransportErrorCode {
resultErr.Code = transientTransportErrorCode
}
case isConnectionLifecycleError(err):
// Preserve lifecycle classification for MarkResult without making the error
// request-scoped (which would also stop credential fallback).
if resultErr.Code == "" || resultErr.Code == connectionLifecycleErrorCode {
resultErr.Code = connectionLifecycleErrorCode
}
case isTransientTransportError(err):
if resultErr.Code == "" || resultErr.Code == transientTransportErrorCode {
resultErr.Code = transientTransportErrorCode
}
}
return resultErr
}
// shouldSkipCredentialCooldown reports failures that must not mark auth/model cooling.
// Connection lifecycle is intentionally separate from request_scoped so transport
// drops do not also stop credential rotation via isRequestInvalidError.
func shouldSkipCredentialCooldown(err *Error) bool {
if err != nil && err.Code == ErrorCodeForceCooldown {
return false
}
return isRequestScopedResultError(err) || isConnectionLifecycleResultError(err) || isTransientTransportResultError(err) || isStreamTruncatedResultError(err)
}
// isConnectionLifecycleError reports transport/session lifecycle failures that must
// not cool credentials: client cancellation and WebSocket/HTTP/2 close/EOF disconnects.
func isConnectionLifecycleError(err error) bool {
if err == nil {
return false
}
// Typed WebSocket close codes are an unambiguous connection lifecycle signal.
var closeErr *websocket.CloseError
if errors.As(err, &closeErr) && closeErr != nil {
switch closeErr.Code {
case websocket.CloseNormalClosure, websocket.CloseGoingAway, websocket.CloseAbnormalClosure:
return true
}
}
// Credential/auth/quota statuses must never be reclassified from response text.
if statusCodeFromError(err) != 0 {
return false
}
if isConfiguredTLSHandshakeTimeoutError(err) {
return false
}
// Client abort and request-scoped timeouts are not credential faults.
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
return true
}
return isConnectionLifecycleMessage(err.Error())
}
func isConnectionLifecycleResultError(err *Error) bool {
if err == nil {
return false
}
if err.Code == connectionLifecycleErrorCode {
return true
}
if err.Code == transientTransportErrorCode {
return false
}
// Message fallback only when no HTTP status is attached, so 401/429/5xx
// response bodies cannot suppress credential cooldown.
if statusCodeFromResult(err) != 0 {
return false
}
return isConnectionLifecycleMessage(err.Message)
}
func isConnectionLifecycleMessage(message string) bool {
lower := strings.ToLower(strings.TrimSpace(message))
if lower == "" {
return false
}
switch lower {
case "context canceled", "context deadline exceeded", "eof", "unexpected eof":
return true
}
// gorilla/websocket CloseError.Error() and common wrappers.
if strings.Contains(lower, "websocket: close 1000") ||
strings.Contains(lower, "websocket: close 1001") ||
strings.Contains(lower, "websocket: close 1006") {
return true
}
// Wrapped transport disconnects (e.g. "read tcp ...: unexpected EOF").
if strings.Contains(lower, "unexpected eof") || isHTTP2ClosedConnectionMessage(lower) {
return true
}
return false
}
func isHTTP2ClosedConnectionMessage(lower string) bool {
// x/net/http2 keeps these errors unexported; allow common wrappers such as url.Error.
return strings.Contains(lower, "http2: client connection force closed via clientconn.close") ||
strings.Contains(lower, "http2: client conn is closed")
}
// isConfiguredTLSHandshakeTimeoutError identifies the typed internal
// establishment deadline produced by the configured TLS handshake bound.
// Delegating to the shared executor predicate ensures classification is
// based on error identity rather than message text and that any
// caller-owned context error outside a configured-timeout subtree wins.
func isConfiguredTLSHandshakeTimeoutError(err error) bool {
if err == nil || statusCodeFromError(err) != 0 {
return false
}
return cliproxyexecutor.IsConfiguredTLSHandshakeTimeoutError(err)
}
// isTransientTransportError reports pre-HTTP dial/TLS/DNS/reset failures that
// should retry under request-retry without cooling the selected credential.
func isTransientTransportError(err error) bool {
if err == nil {
return false
}
// HTTP-status failures stay on the credential/status retry path.
if statusCodeFromError(err) != 0 {
return false
}
if isConfiguredTLSHandshakeTimeoutError(err) {
return true
}
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return false
}
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
return true
}
var dnsErr *net.DNSError
if errors.As(err, &dnsErr) && dnsErr != nil && (dnsErr.IsTimeout || dnsErr.IsTemporary || dnsErr.Timeout()) {
return true
}
var netErr net.Error
if errors.As(err, &netErr) && netErr != nil && netErr.Timeout() {
return true
}
if isTransientSyscallError(err) {
return true
}
var opErr *net.OpError
if errors.As(err, &opErr) && opErr != nil {
return true
}
return isTransientTransportMessage(err.Error())
}
func isTransientTransportResultError(err *Error) bool {
if err == nil {
return false
}
if err.Code == transientTransportErrorCode {
return true
}
if statusCodeFromResult(err) != 0 {
return false
}
// Once an error has been reduced to a Result message, typed identity is
// no longer available. An explicit transient code assigned by
// resultErrorFromError is authoritative for configured handshake
// timeouts; matching text alone must not promote an external or
// caller-owned deadline that happens to use the same phrase.
if strings.Contains(strings.ToLower(err.Message), "tls handshake timeout") {
return false
}
return isTransientTransportMessage(err.Message)
}
func isTransientSyscallError(err error) bool {
var errno syscall.Errno
if !errors.As(err, &errno) {
return false
}
switch errno {
case syscall.ECONNREFUSED, syscall.ECONNRESET, syscall.ECONNABORTED,
syscall.ETIMEDOUT, syscall.EHOSTUNREACH, syscall.ENETUNREACH, syscall.EPIPE:
return true
default:
return false
}
}
func isTransientTransportMessage(message string) bool {
lower := strings.ToLower(strings.TrimSpace(message))
if lower == "" {
return false
}
switch {
case strings.Contains(lower, "tls: tls handshake"),
strings.Contains(lower, "tls handshake timeout"),
strings.Contains(lower, "wsarecv"),
strings.Contains(lower, "wsasend"),
strings.Contains(lower, "a connection attempt failed"),
strings.Contains(lower, "connection refused"),
strings.Contains(lower, "connection reset"),
strings.Contains(lower, "i/o timeout"),
strings.Contains(lower, "no such host"),
strings.Contains(lower, "server misbehaving"),
strings.Contains(lower, "network is unreachable"),
strings.Contains(lower, "no route to host"),
strings.Contains(lower, "broken pipe"),
strings.Contains(lower, "connection aborted"),
strings.Contains(lower, "use of closed network connection"),
strings.Contains(lower, "unexpected eof"),
isHTTP2ClosedConnectionMessage(lower):
return true
default:
return false
}
}
func isUnauthorizedError(err error) bool {
if err == nil || codex.IsRefreshPreSendError(err) {
return false
}
if statusCodeFromError(err) == http.StatusUnauthorized {
return true
}
raw := strings.ToLower(err.Error())
return strings.Contains(raw, "status 401") || strings.Contains(raw, "401 unauthorized") || strings.Contains(raw, "(401)")
}
func hasUnauthorizedAuthFailure(auth *Auth) bool {
if auth == nil || auth.LastError == nil {
return false
}
if auth.Unavailable && auth.Status == StatusError && auth.NextRefreshAfter.IsZero() && auth.NextRetryAfter.IsZero() &&
(auth.LastError.StatusCode() == http.StatusUnauthorized || strings.EqualFold(auth.LastError.Code, "unauthorized")) {
return true
}
return false
}
// HasUnauthorizedAuthFailure reports whether the auth has a terminal unauthorized error
// with no pending refresh scheduled.
func HasUnauthorizedAuthFailure(auth *Auth) bool {
return hasUnauthorizedAuthFailure(auth)
}
func hasDisabledInvalidGrantFailure(auth *Auth) bool {
if auth == nil {
return false
}
isDisabled := auth.Disabled || auth.Status == StatusDisabled
if !isDisabled {
return false
}
if auth.LastError != nil && (isInvalidGrantResultError(auth.LastError) || isInvalidGrantErrorMessage(auth.LastError.Message) || isInvalidGrantErrorMessage(auth.LastError.Code)) {
return true
}
return false
}
// HasDisabledInvalidGrantFailure reports whether the auth is disabled and has encountered an invalid_grant error.
func HasDisabledInvalidGrantFailure(auth *Auth) bool {
return hasDisabledInvalidGrantFailure(auth)
}
var (
renewalURLPattern = regexp.MustCompile(`(?i)https?://[^\s"'<>\\]+`)
renewalJWTPattern = regexp.MustCompile(`\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)?\b`)
renewalSecretKVPattern = regexp.MustCompile(`(?i)\b(access_token|refresh_token|id_token|client_secret|code_verifier|code_challenge|authorization)\s*([=:]|"\s*:\s*")\s*(?:(?:bearer|basic|token)\s+)?([^\s"&',}\\]+)`)
renewalOAuthParamPatten = regexp.MustCompile(`(?i)(?:[?&]|\b)(code|state)=([^\s"&',}\\]+)`)
renewalEscapedParamPattern = regexp.MustCompile(`(?i)(code|state)%3[dD]([^\s"&',}\\%]+)`)
renewalJSONStatePattern = regexp.MustCompile(`(?i)("state"\s*:\s*)"([^"]*)(?:"|$)`)
renewalJSONCodePattern = regexp.MustCompile(`(?i)("code"\s*:\s*)"([^"]*)(?:"|$)`)
renewalBearerPattern = regexp.MustCompile(`(?i)\bBearer\s+[^\s"',}\\]+`)
)
func isKnownSafeOAuthErrorCode(code string) bool {
switch strings.ToLower(strings.TrimSpace(code)) {
case "refresh_outcome_unknown",
"refresh_token_reused",
"refresh_token_expired",
"refresh_token_invalidated",
"token_revoked",
"invalid_grant",
"invalid_client",
"invalid_request",
"invalid_request_error",
"unauthorized",
"rate_limit_exceeded",
"insufficient_quota",
"billing_hard_limit_reached",
"account_deactivated",
"context_length_exceeded",
"server_error",
"service_unavailable",
"bad_gateway",
"model_not_found":
return true
default:
return false
}
}
// SanitizeRenewalMessage strips URLs, bearer tokens, JWTs, OAuth codes/states, and secret fields from diagnostic messages.
func SanitizeRenewalMessage(raw string) string {
return sanitizeRenewalMessage(raw)
}
func redactRenewalPatterns(in string) string {
out := renewalURLPattern.ReplaceAllString(in, "[redacted-url]")
out = renewalJWTPattern.ReplaceAllString(out, "[redacted-token]")
out = renewalBearerPattern.ReplaceAllString(out, "Bearer [redacted]")
out = renewalSecretKVPattern.ReplaceAllString(out, "${1}=[redacted]")
out = renewalOAuthParamPatten.ReplaceAllString(out, "${1}=[redacted]")
out = renewalEscapedParamPattern.ReplaceAllString(out, "${1}=[redacted]")
out = renewalJSONStatePattern.ReplaceAllString(out, `${1}"[redacted]"`)
out = renewalJSONCodePattern.ReplaceAllStringFunc(out, func(match string) string {
sub := renewalJSONCodePattern.FindStringSubmatch(match)
if len(sub) == 3 && isKnownSafeOAuthErrorCode(sub[2]) && strings.HasSuffix(match, `"`) {
return match
}
if len(sub) >= 2 {
return sub[1] + `"[redacted]"`
}
return `"code":"[redacted]"`
})
return out
}
func sanitizeRenewalMessage(raw string) string {
// Redact secrets on the full input BEFORE truncation so secrets straddling the truncation boundary cannot leak.
preRedacted := redactRenewalPatterns(raw)
out := internallogging.SafeDiagnosticForLog(preRedacted)
return redactRenewalPatterns(out)
}
func isTerminalRefreshGrantMessage(message string) bool {
lower := strings.ToLower(message)
for _, token := range []string{
"refresh_outcome_unknown",
"refresh_token_reused",
"refresh_token_expired",
// Anthropic returns {"error": "invalid_grant", "error_description": "Refresh token expired"}.
"refresh token expired",
"refresh_token_invalidated",
"token_revoked",
"invalid_client",
} {
if strings.Contains(lower, token) {
return true
}
}
return false
}
func isTerminalRefreshError(auth *Auth, err error) bool {
if err == nil || codex.IsRefreshPreSendError(err) {
return false
}
var partErr interface {
OuterRefreshError() error
SIWCRefreshError() error
}
if errors.As(err, &partErr) && partErr != nil {
return isTerminalRefreshError(auth, partErr.OuterRefreshError()) ||
isTerminalRefreshError(auth, partErr.SIWCRefreshError())
}
var authErr *Error
if errors.As(err, &authErr) && authErr != nil && isTerminalRefreshGrantMessage(authErr.Code) {
return true
}
if isUnauthorizedError(err) || isTerminalRefreshGrantMessage(err.Error()) {
return true
}
if auth != nil && strings.EqualFold(strings.TrimSpace(auth.Provider), "codex") && isInvalidGrantError(err) {
return true
}
return false
}
func terminalRefreshErrorCode(message string, statusCode int) string {
lower := strings.ToLower(message)
for _, code := range []string{
"refresh_outcome_unknown",
"refresh_token_reused",
"refresh_token_expired",
"refresh_token_invalidated",
"token_revoked",
"invalid_client",
} {
if strings.Contains(lower, code) {
return code
}
}
if strings.Contains(lower, "refresh token expired") {
return "refresh_token_expired"
}
if isInvalidGrantErrorMessage(lower) {
return "invalid_grant"
}
if statusCode == http.StatusUnauthorized {
return "unauthorized"
}
return "refresh_failed"
}
func sanitizeRenewalError(auth *Auth, err error) *Error {
if err == nil {
return nil
}
var partErr interface {
OuterRefreshError() error
SIWCRefreshError() error
}
if errors.As(err, &partErr) && partErr != nil {
outerErr := partErr.OuterRefreshError()
siwcErr := partErr.SIWCRefreshError()
if outerErr != nil && isTerminalRefreshError(auth, outerErr) {
err = outerErr
} else if siwcErr != nil && isTerminalRefreshError(auth, siwcErr) {
err = siwcErr
}
}
raw := err.Error()
statusCode := statusCodeFromError(err)
if statusCode == 0 {
if isUnauthorizedError(err) {
statusCode = http.StatusUnauthorized
} else if isInvalidGrantError(err) || isTerminalRefreshGrantMessage(raw) {
statusCode = http.StatusBadRequest
}
}
terminal := isTerminalRefreshError(auth, err)
code := terminalRefreshErrorCode(raw, statusCode)
var existingAuthErr *Error
if errors.As(err, &existingAuthErr) && existingAuthErr != nil && isKnownSafeOAuthErrorCode(existingAuthErr.Code) {
code = strings.ToLower(strings.TrimSpace(existingAuthErr.Code))
}
return &Error{
Code: code,
Message: sanitizeRenewalMessage(raw),
HTTPStatus: statusCode,
Retryable: !terminal,
}
}
func refreshErrorFromError(err error) *Error {
if err == nil {
return nil
}
statusCode := statusCodeFromError(err)
if statusCode == 0 && isUnauthorizedError(err) {
statusCode = http.StatusUnauthorized
}
authErr := &Error{Message: sanitizeRenewalMessage(err.Error()), HTTPStatus: statusCode}
if statusCode == http.StatusUnauthorized {
authErr.Code = "unauthorized"
authErr.Retryable = false
}
return authErr
}
func retryAfterFromError(err error) *time.Duration {
if err == nil {
return nil
}
// An exhausted Home round owns the aggregated multi-credential deadline
// for scheduling just as it does for response headers.
var exhausted *homeRetryRoundExhaustedError
if errors.As(err, &exhausted) && exhausted != nil {
// The aggregate owns this error even when RetryAfter() is nil: nil
// encodes an explicit zero (immediate) or absent aggregate and must
// not be replaced by the last attempt's positive hint. Callers that
// need the immediate signal consult retryNow / the aggregate first.
return exhausted.RetryAfter()
}
// A locally trusted deadline outranks any hint on the wrapped error so a
// re-recorded terminal error cannot reintroduce a stale upstream value.
if trusted := trustedRetryAfterFromError(err); trusted != nil {
return trusted
}
type retryAfterProvider interface {
RetryAfter() *time.Duration
}
var rap retryAfterProvider
if !errors.As(err, &rap) || rap == nil {
return nil
}
// An expired trusted wrapper reports a negative remaining time; natural
// expiry is not an invalid provider hint, so fall through to the wrapped
// error's own hint instead of surfacing the negative value.
switch rap.(type) {
case *poolCooldownTerminalError, *trustedRetryAfterError:
if inner := errors.Unwrap(err); inner != nil {
return retryAfterFromError(inner)
}
return nil
}
retryAfter := rap.RetryAfter()
if retryAfter == nil {
return nil
}
value := *retryAfter
return &value
}
// isTransientRateLimitError reports a transient per-model capacity/admission
// 429 whose quota is not exhausted. The classification travels through the
// shared executor contract, so only executors that explicitly vouch for the
// failure opt out of quota cooldown handling.
func isTransientRateLimitError(err error) bool {
if err == nil {
return false
}
// A credential-scoped failure always stays on the quota path; a transient
// classification must never downgrade whole-credential exhaustion.
if isCredentialScopedError(err) {
return false
}
transientErr, ok := errors.AsType[cliproxyexecutor.TransientRateLimitError](err)
if ok && transientErr != nil && transientErr.IsTransientRateLimit() {
return true
}
var authErr *Error
if errors.As(err, &authErr) && authErr != nil && authErr.Code == transientRateLimitErrorCode {
return true
}
return false
}
func isTransientRateLimitResultError(err *Error) bool {
return err != nil && err.Code == transientRateLimitErrorCode
}
func isCredentialScopedError(err error) bool {
if err == nil {
return false
}
type credentialScopedProvider interface {
IsCredentialScoped() bool
}
var csp credentialScopedProvider
if errors.As(err, &csp) && csp != nil && csp.IsCredentialScoped() {
return true
}
return strings.EqualFold(errorCodeFromError(err), credentialQuotaErrorCode)
}
func statusCodeFromResult(err *Error) int {
if err == nil {
return 0
}
return err.StatusCode()
}
func isModelSupportErrorMessage(message string) bool {
lower := strings.ToLower(strings.TrimSpace(message))
if lower == "" {
return false
}
patterns := [...]string{
"model_not_supported",
"requested model is not supported",
"requested model is unsupported",
"requested model is unavailable",
"model is not supported",
"model not supported",
"unsupported model",
"model unavailable",
"not available for your plan",
"not available for your account",
}
for _, pattern := range patterns {
if strings.Contains(lower, pattern) {
return true
}
}
return false
}
func isModelSupportError(err error) bool {
if err == nil {
return false
}
if isExplicitModelNotFoundError(err, "") {
return true
}
status := statusCodeFromError(err)
if status != http.StatusBadRequest && status != http.StatusUnprocessableEntity && status != http.StatusNotFound {
return false
}
return isModelSupportErrorMessage(err.Error())
}
func isInvalidGrantErrorMessage(message string) bool {
return strings.Contains(strings.ToLower(message), "invalid_grant")
}
func isInvalidGrantError(err error) bool {
if err == nil || codex.IsRefreshPreSendError(err) {
return false
}
if !isInvalidGrantErrorMessage(err.Error()) {
return false
}
status := statusCodeFromError(err)
if status == http.StatusBadRequest || status == http.StatusUnauthorized || status == 0 {
return true
}
return false
}
func isInvalidGrantResultError(err *Error) bool {
if err == nil {
return false
}
if !isInvalidGrantErrorMessage(err.Code) && !isInvalidGrantErrorMessage(err.Message) {
return false
}
status := statusCodeFromResult(err)
if status == http.StatusBadRequest || status == http.StatusUnauthorized || status == 0 {
return true
}
return false
}
func isCreditBalanceTooLowError(err error) bool {
if err == nil {
return false
}
status := statusCodeFromError(err)
if clienterror.IsCreditBalanceTooLow(status, err) {
return true
}
var authErr *Error
if errors.As(err, &authErr) && authErr != nil && authErr.Message != "" {
return clienterror.IsCreditBalanceTooLow(status, errors.New(authErr.Message))
}
return false
}
func isCreditBalanceTooLowResultError(err *Error) bool {
if err == nil {
return false
}
status := statusCodeFromResult(err)
if clienterror.IsCreditBalanceTooLow(status, err) {
return true
}
if err.Message != "" {
return clienterror.IsCreditBalanceTooLow(status, errors.New(err.Message))
}
return false
}
func isModelSupportResultError(err *Error) bool {
if err == nil {
return false
}
if isExplicitModelNotFoundError(err, "") {
return true
}
status := statusCodeFromResult(err)
if status != http.StatusBadRequest && status != http.StatusUnprocessableEntity && status != http.StatusNotFound {
return false
}
return isModelSupportErrorMessage(err.Message)
}
func isCloudflareChallengeErrorMessage(message string) bool {
lower := strings.ToLower(strings.TrimSpace(message))
return strings.Contains(lower, "challenge-platform") ||
strings.Contains(lower, "cf-mitigated") ||
strings.Contains(lower, "cloudflare challenge") ||
(strings.Contains(lower, "just a moment") && strings.Contains(lower, "cloudflare"))
}
// isCloudflareChallengeError checks whether err is a Cloudflare bot/waf challenge.
// Cloudflare challenges are served with HTTP 403. HTTP status >= 500 indicates an
// upstream gateway/origin failure (such as 520-526 origin errors) and takes precedence
// over challenge classification.
func isCloudflareChallengeError(err error) bool {
if err == nil {
return false
}
if status := statusCodeFromError(err); status >= 500 {
return false
}
return isCloudflareChallengeErrorMessage(err.Error())
}
// isCloudflareChallengeResultError checks whether err is a Cloudflare bot/waf challenge.
// Responses with HTTP status >= 500 represent upstream gateway/origin failures
// (such as Cloudflare 520-526) and are excluded from challenge classification.
func isCloudflareChallengeResultError(err *Error) bool {
if err == nil {
return false
}
if status := statusCodeFromResult(err); status >= 500 {
return false
}
return isCloudflareChallengeErrorMessage(err.Message)
}
// isTOSViolationResultError reports whether the failure is a terminal Antigravity
// Terms-of-Service suspension (403 PERMISSION_DENIED + ErrorInfo reason=
// TOS_VIOLATION). The executor marks these via the tos_violation error code so
// the credential is deactivated instead of cooled; sibling credentials are not
// hammered during an automated sweep.
// isRefreshClearableCredentialError reports whether a credential-scoped non-429
// failure is an ordinary auth/grant error that an OAuth token refresh may clear.
func isRefreshClearableCredentialError(err *Error) bool {
if err == nil || isForceCooldownError(err) {
return false
}
return isUnauthorizedError(err) || isInvalidGrantResultError(err)
}
func isTOSViolationResultError(err *Error) bool {
if err == nil {
return false
}
return strings.EqualFold(strings.TrimSpace(err.Code), "tos_violation")
}
// persistTOSAppealMetadata copies the appeal detail carried on a TOS_VIOLATION
// result error onto the credential record so the management API can surface it
// for the operator. It is a no-op for errors without appeal metadata.
func persistTOSAppealMetadata(auth *Auth, err *Error) {
if auth == nil || err == nil || len(err.Metadata) == 0 {
return
}
if auth.Metadata == nil {
auth.Metadata = make(map[string]any)
}
for key, value := range err.Metadata {
if trimmed := strings.TrimSpace(value); trimmed != "" {
auth.Metadata[key] = trimmed
}
}
}
func nextCloudflareCooldown(backoffLevel int, disableCooling bool, now time.Time) (time.Time, int) {
var next time.Time
if !disableCooling {
cooldown, nextLevel := nextQuotaCooldown(backoffLevel, disableCooling)
if cooldown < 10*time.Second {
cooldown = 10 * time.Second
}
if cooldown > 0 {
next = now.Add(cooldown)
}
backoffLevel = nextLevel
}
return next, backoffLevel
}
func isRequestScopedNotFoundResultError(err *Error) bool {
if err == nil || statusCodeFromResult(err) != http.StatusNotFound {
return false
}
return clienterror.IsItemNotPersisted(err.Message)
}
func isRequestScopedResultError(err *Error) bool {
if err == nil {
return false
}
if err.IsRequestScoped() || isRequestScopedNotFoundResultError(err) {
return true
}
return isRequestInvalidError(err)
}
func isCountTokensEndpointNotFoundError(err error, requestedModel string) bool {
if err == nil {
return false
}
baseModel := thinking.ParseSuffix(requestedModel).ModelName
if isExplicitModelNotFoundError(err, baseModel) {
return false
}
status := statusCodeFromError(err)
if status == http.StatusNotFound {
return true
}
if isCredentialScopedError(err) || isCloudflareChallengeError(err) || isInvalidGrantError(err) || isCreditBalanceTooLowError(err) || isModelSupportErrorMessage(err.Error()) {
return false
}
switch status {
case http.StatusNotImplemented:
return true
case 0:
return isCountTokensUnsupportedErrorCode(errorCodeFromError(err)) || errors.Is(err, errors.ErrUnsupported)
default:
return false
}
}
func isCountTokensUnsupportedErrorCode(value string) bool {
normalized := strings.NewReplacer("-", "_", " ", "_").Replace(strings.ToLower(strings.TrimSpace(value)))
switch normalized {
case "not_implemented",
"count_tokens_unsupported",
"count_tokens_not_supported",
"unsupported_operation",
"unsupported_capability",
"method_not_implemented":
return true
default:
return false
}
}
func isResponsesCompactRequest(opts cliproxyexecutor.Options) bool {
return opts.Alt == "responses/compact"
}
func isResponsesCompactRequestFaultError(opts cliproxyexecutor.Options, err error) bool {
if !isResponsesCompactRequest(opts) || err == nil {
return false
}
if isCredentialScopedError(err) || isCloudflareChallengeError(err) || isInvalidGrantError(err) || isCreditBalanceTooLowError(err) {
return false
}
status := statusCodeFromError(err)
if clienterror.IsRequestFault(status, err) {
return true
}
switch status {
case http.StatusBadRequest,
http.StatusNotFound,
http.StatusMethodNotAllowed,
http.StatusConflict,
http.StatusRequestEntityTooLarge,
http.StatusUnprocessableEntity,
http.StatusNotImplemented:
return true
default:
return false
}
}
func isResponsesCompactAvailabilityNeutralError(opts cliproxyexecutor.Options, err error, resultErr *Error) bool {
if !isResponsesCompactRequest(opts) {
return false
}
if resultErr != nil && resultErr.Code == ErrorCodeForceCooldown {
return false
}
if isCredentialScopedError(err) || isCloudflareChallengeError(err) || isInvalidGrantError(err) || isCreditBalanceTooLowError(err) {
return false
}
if resultErr != nil && (isCloudflareChallengeResultError(resultErr) || isInvalidGrantResultError(resultErr) || isCreditBalanceTooLowResultError(resultErr)) {
return false
}
status := statusCodeFromError(err)
if status == 0 && resultErr != nil {
status = statusCodeFromResult(resultErr)
}
if status == http.StatusUnauthorized || status == http.StatusPaymentRequired || status == http.StatusForbidden || status == http.StatusTooManyRequests {
return false
}
return true
}
func isExplicitModelNotFoundError(err error, requestedModel string) bool {
if err == nil {
return false
}
if isModelNotFoundIdentifier(errorCodeFromError(err)) {
return true
}
var authErr *Error
if errors.As(err, &authErr) && authErr != nil {
if isModelNotFoundIdentifier(authErr.Code) || isStructuredModelNotFoundError(authErr.Message, requestedModel) || isStructuredModelNotFoundError(authErr.Error(), requestedModel) {
return true
}
} else if isStructuredModelNotFoundError(err.Error(), requestedModel) {
return true
}
switch wrapped := err.(type) {
case interface{ Unwrap() []error }:
for _, nested := range wrapped.Unwrap() {
if isExplicitModelNotFoundError(nested, requestedModel) {
return true
}
}
case interface{ Unwrap() error }:
return isExplicitModelNotFoundError(wrapped.Unwrap(), requestedModel)
}
return false
}
func isStructuredModelNotFoundError(message, requestedModel string) bool {
var payload any
if errJSON := json.Unmarshal([]byte(strings.TrimSpace(message)), &payload); errJSON != nil {
return false
}
return containsStructuredModelNotFound(payload, requestedModel)
}
func containsStructuredModelNotFound(value any, requestedModel string) bool {
switch typed := value.(type) {
case map[string]any:
notFoundType := false
exactModelReference := false
for key, item := range typed {
text, isString := item.(string)
if isString {
switch strings.ToLower(strings.TrimSpace(key)) {
case "code":
if isModelNotFoundIdentifier(text) {
return true
}
case "type":
if isModelNotFoundIdentifier(text) {
return true
}
notFoundType = notFoundType || isNotFoundErrorIdentifier(text)
case "error", "message", "detail", "error_description", "title":
if isExplicitModelNotFoundMessage(text, requestedModel) {
return true
}
exactModelReference = exactModelReference || isExactRequestedModelReference(text, requestedModel)
}
}
switch item.(type) {
case map[string]any, []any:
if containsStructuredModelNotFound(item, requestedModel) {
return true
}
}
}
return notFoundType && exactModelReference
case []any:
for _, item := range typed {
if text, isString := item.(string); isString && isExplicitModelNotFoundMessage(text, requestedModel) {
return true
}
if containsStructuredModelNotFound(item, requestedModel) {
return true
}
}
}
return false
}
func isModelNotFoundIdentifier(value string) bool {
candidate := strings.ToLower(strings.TrimSpace(value))
if fragment := strings.LastIndex(candidate, "#"); fragment >= 0 && fragment+1 < len(candidate) {
candidate = candidate[fragment+1:]
} else {
if query := strings.Index(candidate, "?"); query >= 0 {
candidate = candidate[:query]
}
candidate = strings.TrimRight(candidate, "/")
if separator := strings.LastIndexAny(candidate, "/:"); separator >= 0 {
candidate = candidate[separator+1:]
}
}
normalized := strings.NewReplacer("-", "_", " ", "_").Replace(candidate)
switch normalized {
case "model_not_found", "model_not_found_error", "unknown_model", "model_does_not_exist", "model_not_exist":
return true
default:
return false
}
}
func isNotFoundErrorIdentifier(value string) bool {
normalized := strings.NewReplacer("-", "_", " ", "_").Replace(strings.ToLower(strings.TrimSpace(value)))
return normalized == "not_found" || normalized == "not_found_error"
}
func isExplicitModelNotFoundMessage(message, requestedModel string) bool {
lower := strings.Trim(strings.ToLower(strings.TrimSpace(message)), " .!;\t\r\n")
if lower == "" {
return false
}
if strings.Contains(lower, "in request") || strings.Contains(lower, "in body") || strings.Contains(lower, "request body") {
return false
}
normalized := strings.NewReplacer("-", "_").Replace(lower)
if strings.Contains(normalized, "model_not_found") || strings.Contains(normalized, "unknown_model") {
return true
}
for _, prefix := range []string{"no such model", "unknown model"} {
if lower != prefix && !strings.HasPrefix(lower, prefix+" ") && !strings.HasPrefix(lower, prefix+":") {
continue
}
remainder := strings.TrimSpace(strings.TrimPrefix(lower, prefix))
remainder = strings.TrimSpace(strings.TrimPrefix(remainder, ":"))
if remainder == "" {
return true
}
missingSuffix, matches := trimRequestedModelReference(remainder, requestedModel)
return matches && missingSuffix == ""
}
for _, prefix := range []string{"the requested model", "requested model", "the model", "model"} {
if lower != prefix && !strings.HasPrefix(lower, prefix+" ") && !strings.HasPrefix(lower, prefix+":") {
continue
}
remainder := strings.TrimSpace(strings.TrimPrefix(lower, prefix))
remainder = strings.TrimSpace(strings.TrimPrefix(remainder, ":"))
if isMissingModelPhrase(remainder) {
return true
}
missingSuffix, matches := trimRequestedModelReference(remainder, requestedModel)
return matches && isMissingModelPhrase(missingSuffix)
}
return false
}
func isExactRequestedModelReference(message, requestedModel string) bool {
lower := strings.Trim(strings.ToLower(strings.TrimSpace(message)), " .!;\t\r\n")
for _, prefix := range []string{"the requested model", "requested model", "the model", "model"} {
if lower != prefix && !strings.HasPrefix(lower, prefix+" ") && !strings.HasPrefix(lower, prefix+":") {
continue
}
remainder := strings.TrimSpace(strings.TrimPrefix(lower, prefix))
remainder = strings.TrimSpace(strings.TrimPrefix(remainder, ":"))
suffix, matches := trimRequestedModelReference(remainder, requestedModel)
return matches && suffix == ""
}
return false
}
func trimRequestedModelReference(value, requestedModel string) (string, bool) {
model := strings.ToLower(strings.TrimSpace(requestedModel))
if model == "" {
return "", false
}
for _, candidate := range []string{model, "'" + model + "'", `"` + model + `"`, "`" + model + "`"} {
if value == candidate {
return "", true
}
if !strings.HasPrefix(value, candidate) {
continue
}
remainder := value[len(candidate):]
if remainder == "" || strings.ContainsRune(" :,", rune(remainder[0])) {
return strings.TrimLeft(remainder, " :,"), true
}
}
return "", false
}
func isMissingModelPhrase(value string) bool {
switch strings.Trim(value, " .!;\t\r\n") {
case "not found", "was not found", "could not be found", "does not exist", "doesn't exist", "not exist", "is unknown", "does not exist or you do not have access to it":
return true
default:
return false
}
}
// isRequestInvalidError returns true if the error represents a client request
// error that should neither rotate nor penalize credentials. Model-support
// errors remain eligible for alternate routing and keep their model-level state.
func isRequestInvalidError(err error) bool {
if err == nil {
return false
}
if isStreamTruncatedError(err) {
// A truncated accepted stream must stop here: the generation was
// already submitted and must not be replayed on another credential.
return true
}
if isRequestScopedError(err) {
return true
}
if isCloudflareChallengeError(err) {
return false
}
if isInvalidGrantError(err) {
return false
}
if isCreditBalanceTooLowError(err) {
return false
}
if isModelSupportError(err) {
return false
}
status := statusCodeFromError(err)
if clienterror.IsRequestFault(status, err) {
return true
}
var authErr *Error
if errors.As(err, &authErr) && authErr != nil && authErr.Message != "" {
// When authErr.Code is non-empty, Error() formats as "Code: Message" which
// breaks JSON parsing in clienterror. Re-evaluate against the raw Message body.
if clienterror.IsRequestFault(status, errors.New(authErr.Message)) {
return true
}
}
return false
}
func applyAuthFailureState(auth *Auth, resultErr *Error, retryAfter *time.Duration, now time.Time, disableCooling bool) (time.Time, time.Time) {
if auth == nil {
return time.Time{}, time.Time{}
}
seedInitialForceCooldownDeadlines(auth)
prevAuthRetryAfter := auth.NextRetryAfter
prevAuthForceUntil := auth.forceCooldownUntil
prevAuthQuota := cooldownFieldsOf(auth.Quota)
prevAuthQuotaSignals := maps.Clone(auth.Quota.cooldownSignals)
var prevCredQuotaForceUntil time.Time
if prevAuthQuota.Exceeded && prevAuthQuota.Reason == "credential_quota" && prevAuthQuota.NextRecoverAt.After(now) && auth.credQuotaForceUntil.After(now) {
prevCredQuotaForceUntil = auth.credQuotaForceUntil
}
var ownAuthForceDeadline time.Time
var ownAuthNonQuotaDeadline time.Time
isForceAuth := isForceCooldownError(resultErr)
if shouldSkipCredentialCooldown(resultErr) {
return time.Time{}, time.Time{}
}
defer func() {
if disableCooling && auth.NextRetryAfter.IsZero() && auth.Quota.NextRecoverAt.IsZero() {
auth.Unavailable = false
auth.Quota.Exceeded = false
}
}()
auth.Unavailable = true
auth.Status = StatusError
auth.UpdatedAt = now
if resultErr != nil {
auth.LastError = cloneError(resultErr)
if resultErr.Message != "" {
auth.StatusMessage = resultErr.Message
}
}
statusCode := statusCodeFromResult(resultErr)
if isTransientRateLimitResultError(resultErr) && statusCode == http.StatusTooManyRequests {
// Without a model key the transient capacity rejection degrades to an
// auth-level transient cooldown (issue 7017), never quota exhaustion.
next := transientRateLimitRetryAfter(now, retryAfter, disableCooling)
ownAuthNonQuotaDeadline = next
if isForceAuth {
ownAuthForceDeadline = next
}
if !next.IsZero() && auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(next) {
next = auth.Quota.NextRecoverAt
}
if next.IsZero() && auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(now) {
next = auth.Quota.NextRecoverAt
}
auth.NextRetryAfter = next
auth.Unavailable = !next.IsZero()
} else if isCloudflareChallengeResultError(resultErr) {
auth.StatusMessage = "cloudflare challenge"
next, backoffLevel := nextCloudflareCooldown(auth.Quota.BackoffLevel, disableCooling, now)
ownAuthNonQuotaDeadline = next
applyCooldownFields(&auth.Quota, QuotaState{
Exceeded: true,
Reason: "cloudflare challenge",
NextRecoverAt: next,
BackoffLevel: backoffLevel,
})
auth.NextRetryAfter = next
} else if isInvalidGrantResultError(resultErr) {
auth.StatusMessage = "invalid_grant"
if disableCooling {
auth.NextRetryAfter = time.Time{}
} else {
auth.NextRetryAfter = now.Add(30 * time.Minute)
}
} else if isCreditBalanceTooLowResultError(resultErr) {
auth.StatusMessage = "payment_required"
if disableCooling {
auth.NextRetryAfter = time.Time{}
} else {
auth.NextRetryAfter = now.Add(30 * time.Minute)
}
} else {
switch statusCode {
case 401:
auth.StatusMessage = "unauthorized"
if disableCooling {
auth.NextRetryAfter = time.Time{}
} else {
auth.NextRetryAfter = now.Add(30 * time.Minute)
}
case 402, 403:
if isTOSViolationResultError(resultErr) {
// Terminal credential suspension: deactivate so the credential is
// excluded from selection/rotation, but keep OAuth refresh intact
// (the suspension is service-scoped per issue #7515; the token and
// model catalog keep working) so operator tooling can verify
// account liveness. The appeal detail travels on the result error's
// Metadata and is persisted onto the credential record below.
auth.StatusMessage = "tos_violation"
auth.Status = StatusDisabled
auth.Disabled = true
auth.Unavailable = true
auth.NextRetryAfter = time.Time{}
auth.LastError = cloneError(resultErr)
persistTOSAppealMetadata(auth, resultErr)
} else {
auth.StatusMessage = "payment_required"
if disableCooling {
auth.NextRetryAfter = time.Time{}
} else {
auth.NextRetryAfter = now.Add(30 * time.Minute)
}
}
case 404:
auth.StatusMessage = "not_found"
if disableCooling {
auth.NextRetryAfter = time.Time{}
} else if retryAfter != nil && *retryAfter > 0 {
auth.NextRetryAfter = now.Add(*retryAfter)
} else {
auth.NextRetryAfter = now.Add(12 * time.Hour)
}
case 429:
auth.StatusMessage = "quota exhausted"
auth.Quota.Exceeded = true
auth.Quota.Reason = "quota"
var next time.Time
if !disableCooling {
if retryAfter != nil {
cooldown := *retryAfter
if cooldown < minQuotaCooldownFloor {
cooldown = minQuotaCooldownFloor
}
next = now.Add(cooldown).Round(0)
if isForceAuth {
ownAuthForceDeadline = next
}
} else {
reusedOpenAuthWindow := auth.Quota.NextRecoverAt.After(now)
baseAuthBackoff := auth.Quota.BackoffLevel
next, auth.Quota.BackoffLevel = quotaCooldownAfterFailure(auth.Quota, now)
if isForceAuth {
if !reusedOpenAuthWindow {
ownAuthForceDeadline = next
} else if prevAuthForceUntil.After(now) {
ownAuthForceDeadline = prevAuthForceUntil
} else {
ownAuthForceDeadline, _ = quotaCooldownAfterFailure(QuotaState{BackoffLevel: baseAuthBackoff}, now)
}
}
}
if auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(next) {
next = auth.Quota.NextRecoverAt
}
}
auth.Quota.NextRecoverAt = next
auth.NextRetryAfter = next
case 408, 500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526:
auth.StatusMessage = "transient upstream error"
auth.NextRetryAfter = recoverableFailureRetryAfterWithHint(now, retryAfter, disableCooling)
auth.Unavailable = !auth.NextRetryAfter.IsZero()
default:
if auth.StatusMessage == "" {
auth.StatusMessage = "request failed"
}
auth.NextRetryAfter = recoverableFailureRetryAfter(now, disableCooling)
auth.Unavailable = !auth.NextRetryAfter.IsZero()
}
}
if isForceAuth && auth.NextRetryAfter.IsZero() {
auth.NextRetryAfter = now.Add(transientErrorCooldown)
auth.Unavailable = true
}
ownAuthRetryDeadline := auth.NextRetryAfter
if isForceAuth && ownAuthForceDeadline.After(ownAuthRetryDeadline) {
ownAuthRetryDeadline = ownAuthForceDeadline
}
if !isQuotaCooldownResultError(resultErr) {
if !ownAuthNonQuotaDeadline.IsZero() || isTransientRateLimitResultError(resultErr) {
ownAuthRetryDeadline = ownAuthNonQuotaDeadline
}
if !isTransientRateLimitResultError(resultErr) && ownAuthNonQuotaDeadline.IsZero() {
ownAuthNonQuotaDeadline = auth.NextRetryAfter
}
if isForceAuth && ownAuthForceDeadline.After(ownAuthNonQuotaDeadline) {
ownAuthNonQuotaDeadline = ownAuthForceDeadline
}
if ownAuthNonQuotaDeadline.After(auth.nonQuotaCooldownUntil) {
auth.nonQuotaCooldownUntil = ownAuthNonQuotaDeadline
}
}
if isForceAuth {
if ownAuthForceDeadline.IsZero() {
ownAuthForceDeadline = auth.NextRetryAfter
}
if ownAuthForceDeadline.After(prevAuthForceUntil) {
auth.forceCooldownUntil = ownAuthForceDeadline
} else {
auth.forceCooldownUntil = prevAuthForceUntil
}
} else {
auth.forceCooldownUntil = prevAuthForceUntil
}
auth.forceCooldownSeeded = true
if isForceAuth && isQuotaCooldownResultError(resultErr) && ownAuthForceDeadline.After(auth.quotaForceCooldownUntil) {
auth.quotaForceCooldownUntil = ownAuthForceDeadline
}
if prevCredQuotaForceUntil.After(now) {
if prevCredQuotaForceUntil.After(auth.credQuotaForceUntil) {
auth.credQuotaForceUntil = prevCredQuotaForceUntil
}
if !auth.Quota.Exceeded || auth.Quota.Reason != "credential_quota" || auth.Quota.NextRecoverAt.Before(prevCredQuotaForceUntil) {
restored := prevAuthQuota
if disableCooling && restored.NextRecoverAt.After(prevCredQuotaForceUntil) {
restored.NextRecoverAt = prevCredQuotaForceUntil
} else if !disableCooling && auth.Quota.Exceeded && auth.Quota.NextRecoverAt.After(restored.NextRecoverAt) {
restored.NextRecoverAt = auth.Quota.NextRecoverAt
}
restored.cooldownSignals = prevAuthQuotaSignals
applyCooldownFields(&auth.Quota, restored)
}
}
activeAuthForce := auth.forceCooldownUntil
if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.credQuotaForceUntil.After(activeAuthForce) {
activeAuthForce = auth.credQuotaForceUntil
}
if activeAuthForce.After(now) {
if auth.NextRetryAfter.Before(activeAuthForce) {
auth.NextRetryAfter = activeAuthForce
}
auth.Unavailable = true
if (!auth.Quota.Exceeded || auth.Quota.NextRecoverAt.IsZero()) && prevAuthQuota.Exceeded && prevAuthQuota.NextRecoverAt.After(now) {
restored := prevAuthQuota
if disableCooling && restored.NextRecoverAt.After(activeAuthForce) {
restored.NextRecoverAt = activeAuthForce
}
restored.cooldownSignals = prevAuthQuotaSignals
applyCooldownFields(&auth.Quota, restored)
}
}
// A later failure only extends a still-live credential cooldown; a
// deliberate zero write (disableCooling) still clears it.
if !auth.NextRetryAfter.IsZero() && prevAuthRetryAfter.After(auth.NextRetryAfter) && prevAuthRetryAfter.After(now) {
auth.NextRetryAfter = prevAuthRetryAfter
}
if !isForceAuth {
return time.Time{}, ownAuthRetryDeadline
}
return ownAuthForceDeadline, ownAuthRetryDeadline
}
// quotaCooldownAfterFailure returns the recovery deadline and backoff level for
// a quota failure observed at now. Failures that land while a previous quota
// window is still open reuse that window instead of escalating, so a burst of
// concurrent in-flight failures advances the backoff ladder at most once per
// window.
func quotaCooldownAfterFailure(quota QuotaState, now time.Time) (time.Time, int) {
if quota.NextRecoverAt.After(now) {
return quota.NextRecoverAt, quota.BackoffLevel
}
cooldown, nextLevel := nextQuotaCooldown(quota.BackoffLevel, false)
var next time.Time
if cooldown > 0 {
next = now.Add(cooldown).Round(0)
}
return next, nextLevel
}
// nextQuotaCooldown returns the next cooldown duration and updated backoff level for repeated quota errors.
func nextQuotaCooldown(prevLevel int, disableCooling bool) (time.Duration, int) {
if prevLevel < 0 {
prevLevel = 0
}
if disableCooling {
return 0, prevLevel
}
cooldown := quotaBackoffBase * time.Duration(1<<prevLevel)
if cooldown < quotaBackoffBase {
cooldown = quotaBackoffBase
}
if cooldown >= quotaBackoffMax {
return quotaBackoffMax, prevLevel
}
return cooldown, prevLevel + 1
}