diff --git a/internal/api/handlers/management/auth_files.go b/internal/api/handlers/management/auth_files.go index a16b36233..37902b942 100644 --- a/internal/api/handlers/management/auth_files.go +++ b/internal/api/handlers/management/auth_files.go @@ -484,6 +484,156 @@ func (h *Handler) buildAuthFileEntry(auth *coreauth.Auth, quotaSupported ...map[ return h.buildAuthFileEntryLocked(auth, quotaSupported...) } +func isPersistentAuthFailure(auth *coreauth.Auth, now time.Time) bool { + if auth == nil { + return false + } + // Terminal unauthorized failure with no refresh scheduled. + if coreauth.HasUnauthorizedAuthFailure(auth) { + return true + } + // An OAuth credential whose access token is expired cannot be used to serve requests. + if exp, ok := auth.AccessTokenExpirationTime(); ok && !exp.IsZero() && !exp.After(now) { + return true + } + // An explicit token expiration status. + if strings.EqualFold(strings.TrimSpace(auth.StatusMessage), "token expired") { + return true + } + return false +} + +func isModelStateBlocked(state *coreauth.ModelState, now time.Time) bool { + if state == nil { + return false + } + if state.Status == coreauth.StatusDisabled { + return true + } + if !state.Unavailable && !state.Quota.Exceeded { + return false + } + hasRecoveryTime := !state.NextRetryAfter.IsZero() || (!state.Quota.NextRecoverAt.IsZero() && state.Quota.Exceeded) + if !state.NextRetryAfter.IsZero() && state.NextRetryAfter.After(now) { + return true + } + if state.Quota.Exceeded && !state.Quota.NextRecoverAt.IsZero() && state.Quota.NextRecoverAt.After(now) { + return true + } + if hasRecoveryTime { + return false + } + return true +} + +func reconcileAuthFileCooldownState(auth *coreauth.Auth, now time.Time) (unavailable bool, status coreauth.Status, statusMessage string, nextRetry time.Time) { + if auth == nil { + return false, coreauth.StatusActive, "", time.Time{} + } + unavailable = auth.Unavailable + status = auth.Status + statusMessage = auth.StatusMessage + if !auth.NextRetryAfter.IsZero() { + nextRetry = auth.NextRetryAfter + } + + if auth.Disabled || auth.Status == coreauth.StatusDisabled { + return unavailable, coreauth.StatusDisabled, statusMessage, nextRetry + } + + // Never reconcile an active authentication or token failure to active. + if isPersistentAuthFailure(auth, now) { + if !nextRetry.IsZero() && !nextRetry.After(now) { + nextRetry = time.Time{} + } + return true, coreauth.StatusError, statusMessage, nextRetry + } + + // Check if there is an active credential-level cooldown. + // Matching selector.availabilityBlock: if neither Unavailable nor Quota.Exceeded is true, + // an inactive timestamp does not block the credential. + hasActiveCredCooldown := false + if auth.Unavailable || auth.Quota.Exceeded { + if !auth.NextRetryAfter.IsZero() && auth.NextRetryAfter.After(now) { + hasActiveCredCooldown = true + } + if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now) { + hasActiveCredCooldown = true + if nextRetry.IsZero() || auth.Quota.NextRecoverAt.After(nextRetry) { + nextRetry = auth.Quota.NextRecoverAt + } + } + } + + // Check per-model states. + hasSchedulableModels := false + allSchedulableBlocked := true + hasActiveModelCooldown := false + hadAnyModelCooldown := false + for _, state := range auth.ModelStates { + if state == nil { + continue + } + if state.Status == coreauth.StatusDisabled { + continue + } + hasSchedulableModels = true + if !state.NextRetryAfter.IsZero() || (state.Quota.Exceeded && !state.Quota.NextRecoverAt.IsZero()) { + hadAnyModelCooldown = true + } + if (!state.NextRetryAfter.IsZero() && state.NextRetryAfter.After(now)) || + (state.Quota.Exceeded && !state.Quota.NextRecoverAt.IsZero() && state.Quota.NextRecoverAt.After(now)) { + hasActiveModelCooldown = true + } + if !isModelStateBlocked(state, now) { + allSchedulableBlocked = false + } + } + + hadCooldown := !auth.NextRetryAfter.IsZero() || + (auth.Quota.Exceeded && !auth.Quota.NextRecoverAt.IsZero()) || + hadAnyModelCooldown + + // If there is an active credential cooldown, keep unavailable/error. + // If all recorded models are blocked and the credential itself was marked unavailable, keep unavailable/error. + if hasActiveCredCooldown || (hasSchedulableModels && allSchedulableBlocked && auth.Unavailable) { + if !nextRetry.IsZero() && !nextRetry.After(now) { + nextRetry = time.Time{} + } + return true, coreauth.StatusError, statusMessage, nextRetry + } + + // If the credential was not marked unavailable and has no active credential cooldown, keep unavailable=false. + if !auth.Unavailable && !hasActiveCredCooldown { + if status == coreauth.StatusError && hasSchedulableModels && !allSchedulableBlocked { + status = coreauth.StatusActive + statusMessage = "" + } + return false, status, statusMessage, time.Time{} + } + + // If a cooldown was recorded but has expired (and no active model cooldown blocks all models): + if hadCooldown && !hasActiveCredCooldown && !hasActiveModelCooldown { + return false, coreauth.StatusActive, "", time.Time{} + } + + // If partial models are still cooling, the credential as a whole remains available for other models. + if hadCooldown && hasSchedulableModels && !allSchedulableBlocked { + if status == coreauth.StatusError && !hasActiveCredCooldown { + status = coreauth.StatusActive + statusMessage = "" + } + return false, status, statusMessage, time.Time{} + } + + // If nextRetry is in the past, do not expose a past retry deadline. + if !nextRetry.IsZero() && !nextRetry.After(now) { + nextRetry = time.Time{} + } + + return unavailable, status, statusMessage, nextRetry +} + func (h *Handler) buildAuthFileEntryLocked(auth *coreauth.Auth, quotaSupported ...map[string]struct{}) gin.H { if auth == nil { return nil @@ -501,6 +651,7 @@ func (h *Handler) buildAuthFileEntryLocked(auth *coreauth.Auth, quotaSupported . if name == "" { name = auth.ID } + unavailable, status, statusMessage, nextRetryAfter := reconcileAuthFileCooldownState(auth, time.Now().UTC()) entry := gin.H{ "id": auth.ID, "auth_index": auth.Index, @@ -508,10 +659,10 @@ func (h *Handler) buildAuthFileEntryLocked(auth *coreauth.Auth, quotaSupported . "type": strings.TrimSpace(auth.Provider), "provider": strings.TrimSpace(auth.Provider), "label": auth.Label, - "status": auth.Status, - "status_message": auth.StatusMessage, + "status": status, + "status_message": statusMessage, "disabled": auth.Disabled, - "unavailable": auth.Unavailable, + "unavailable": unavailable, "runtime_only": runtimeOnly, "source": "memory", "size": int64(0), @@ -571,8 +722,8 @@ func (h *Handler) buildAuthFileEntryLocked(auth *coreauth.Auth, quotaSupported . if !auth.LastRefreshedAt.IsZero() { entry["last_refresh"] = auth.LastRefreshedAt } - if !auth.NextRetryAfter.IsZero() { - entry["next_retry_after"] = auth.NextRetryAfter + if !nextRetryAfter.IsZero() { + entry["next_retry_after"] = nextRetryAfter } if path != "" { entry["path"] = path diff --git a/internal/api/handlers/management/auth_files_cooldown_test.go b/internal/api/handlers/management/auth_files_cooldown_test.go index a601c5b49..ab8e36000 100644 --- a/internal/api/handlers/management/auth_files_cooldown_test.go +++ b/internal/api/handlers/management/auth_files_cooldown_test.go @@ -23,6 +23,7 @@ type authFilesCooldownResponse struct { AuthIndex string `json:"auth_index"` Name string `json:"name"` Status string `json:"status"` + StatusMessage string `json:"status_message"` Unavailable bool `json:"unavailable"` NextRetryAfter time.Time `json:"next_retry_after"` Cooldowns json.RawMessage `json:"cooldowns"` @@ -187,3 +188,576 @@ func TestListAuthFilesCooldownsUnknown(t *testing.T) { }) } } + +func TestListAuthFiles_ExpiredCooldownReconciledToActive(t *testing.T) { + t.Setenv("MANAGEMENT_PASSWORD", "") + now := time.Now().UTC() + expiredDeadline := now.Add(-10 * time.Minute) + + manager := coreauth.NewManager(nil, nil, nil) + cfg := &config.Config{AuthDir: t.TempDir()} + manager.SetConfig(cfg) + + // Credential-level expired cooldown (e.g. Codex quota exhaustion in Issue 5964) + registerAuthForLookupTest(t, manager, &coreauth.Auth{ + ID: "codex-expired", + Index: "idx-codex-expired", + FileName: "codex-expired.json", + Provider: "codex", + Status: coreauth.StatusError, + StatusMessage: "credential_quota", + Unavailable: true, + NextRetryAfter: expiredDeadline, + Quota: coreauth.QuotaState{ + Exceeded: true, + Reason: "credential_quota", + NextRecoverAt: expiredDeadline, + ObservedAt: expiredDeadline, + }, + Attributes: map[string]string{"runtime_only": "true"}, + }) + + // Model-level expired cooldown where all model cooldowns have elapsed + registerAuthForLookupTest(t, manager, &coreauth.Auth{ + ID: "model-expired", + Index: "idx-model-expired", + FileName: "model-expired.json", + Provider: "claude", + Status: coreauth.StatusError, + StatusMessage: "rate limit exceeded", + Unavailable: true, + NextRetryAfter: expiredDeadline, + ModelStates: map[string]*coreauth.ModelState{ + "claude-3-5-sonnet": { + Status: coreauth.StatusError, + StatusMessage: "rate limit exceeded", + Unavailable: true, + NextRetryAfter: expiredDeadline, + }, + }, + Attributes: map[string]string{"runtime_only": "true"}, + }) + + h := NewHandlerWithoutConfigFilePath(cfg, manager) + payload := requestAuthFilesCooldowns(t, h, "") + + if len(payload.Files) != 2 { + t.Fatalf("expected 2 files, got %d", len(payload.Files)) + } + + for _, file := range payload.Files { + if file.Unavailable { + t.Errorf("auth %s: expected unavailable=false after cooldown expiration, got true", file.ID) + } + if file.Status != string(coreauth.StatusActive) { + t.Errorf("auth %s: expected status=%q after cooldown expiration, got %q", file.ID, coreauth.StatusActive, file.Status) + } + if file.StatusMessage != "" { + t.Errorf("auth %s: expected empty status_message after cooldown expiration, got %q", file.ID, file.StatusMessage) + } + if !file.NextRetryAfter.IsZero() { + t.Errorf("auth %s: expected zero next_retry_after after cooldown expiration, got %v", file.ID, file.NextRetryAfter) + } + if string(file.Cooldowns) != "[]" { + t.Errorf("auth %s: expected cooldowns=[], got %s", file.ID, file.Cooldowns) + } + } +} + +func TestListAuthFiles_ExpiredCooldownWithSubsequentTokenFailure(t *testing.T) { + t.Setenv("MANAGEMENT_PASSWORD", "") + now := time.Now().UTC() + expiredDeadline := now.Add(-10 * time.Minute) + + manager := coreauth.NewManager(nil, nil, nil) + cfg := &config.Config{AuthDir: t.TempDir()} + manager.SetConfig(cfg) + + // An OAuth auth that had a cooldown, but subsequently had its access token expire / refresh fail. + registerAuthForLookupTest(t, manager, &coreauth.Auth{ + ID: "codex-token-expired", + Index: "idx-codex-token-expired", + FileName: "codex-token-expired.json", + Provider: "codex", + Status: coreauth.StatusError, + StatusMessage: "token expired", + Unavailable: true, + NextRetryAfter: expiredDeadline, + Metadata: map[string]any{ + "type": "codex", + "access_token": "expired-access-token", + "expired": now.Add(-5 * time.Minute).Format(time.RFC3339), + }, + Attributes: map[string]string{"runtime_only": "true"}, + }) + + h := NewHandlerWithoutConfigFilePath(cfg, manager) + payload := requestAuthFilesCooldowns(t, h, "?name=codex-token-expired.json") + + if len(payload.Files) != 1 { + t.Fatalf("expected 1 file, got %d", len(payload.Files)) + } + + file := payload.Files[0] + if !file.Unavailable { + t.Error("expected unavailable=true for expired token failure, got false") + } + if file.Status != string(coreauth.StatusError) { + t.Errorf("expected status=%q for expired token failure, got %q", coreauth.StatusError, file.Status) + } + if file.StatusMessage != "token expired" { + t.Errorf("expected status_message=%q, got %q", "token expired", file.StatusMessage) + } + if !file.NextRetryAfter.IsZero() { + t.Errorf("expected zero next_retry_after, got %v", file.NextRetryAfter) + } +} + +func TestListAuthFiles_PartialModelCooldownWithAuthFailure(t *testing.T) { + t.Setenv("MANAGEMENT_PASSWORD", "") + now := time.Now().UTC() + activeDeadline := now.Add(10 * time.Minute) + + manager := coreauth.NewManager(nil, nil, nil) + cfg := &config.Config{AuthDir: t.TempDir()} + manager.SetConfig(cfg) + + // Auth has an independent auth error ("unauthorized"), but only one of its models is in cooldown. + registerAuthForLookupTest(t, manager, &coreauth.Auth{ + ID: "auth-unauthorized-partial-model", + Index: "idx-auth-unauthorized", + FileName: "auth-unauthorized.json", + Provider: "claude", + Status: coreauth.StatusError, + StatusMessage: "unauthorized", + Unavailable: true, + LastError: &coreauth.Error{HTTPStatus: 401, Message: "unauthorized"}, + ModelStates: map[string]*coreauth.ModelState{ + "model-cool": { + Status: coreauth.StatusError, + Unavailable: true, + NextRetryAfter: activeDeadline, + }, + "model-free": { + Status: coreauth.StatusActive, + }, + }, + Attributes: map[string]string{"runtime_only": "true"}, + }) + + h := NewHandlerWithoutConfigFilePath(cfg, manager) + payload := requestAuthFilesCooldowns(t, h, "?name=auth-unauthorized.json") + + if len(payload.Files) != 1 { + t.Fatalf("expected 1 file, got %d", len(payload.Files)) + } + + file := payload.Files[0] + if !file.Unavailable { + t.Error("expected unavailable=true due to unauthorized error, got false") + } + if file.Status != string(coreauth.StatusError) { + t.Errorf("expected status=%q, got %q", coreauth.StatusError, file.Status) + } + if file.StatusMessage != "unauthorized" { + t.Errorf("expected status_message=%q, got %q", "unauthorized", file.StatusMessage) + } +} + +func TestListAuthFiles_UnexpiredTokenWithRefresh401_NoCooldown(t *testing.T) { + t.Setenv("MANAGEMENT_PASSWORD", "") + now := time.Now().UTC() + futureExpiry := now.Add(48 * time.Hour).Format(time.RFC3339) + retryBackoff := now.Add(5 * time.Minute) + + manager := coreauth.NewManager(nil, nil, nil) + cfg := &config.Config{AuthDir: t.TempDir()} + manager.SetConfig(cfg) + + // An OAuth credential whose access token is still valid (+48h), but background refresh failed with 401 + // and scheduled a retry backoff. The credential itself is still active and usable. + registerAuthForLookupTest(t, manager, &coreauth.Auth{ + ID: "auth-valid-token-refresh-401", + Index: "idx-valid-token", + FileName: "auth-valid-token.json", + Provider: "codex", + Status: coreauth.StatusActive, + Unavailable: false, + NextRefreshAfter: retryBackoff, + LastError: &coreauth.Error{HTTPStatus: 401, Message: "401 unauthorized on refresh"}, + Metadata: map[string]any{ + "type": "codex", + "access_token": "valid-future-access-token", + "expired": futureExpiry, + }, + Attributes: map[string]string{"runtime_only": "true"}, + }) + + h := NewHandlerWithoutConfigFilePath(cfg, manager) + payload := requestAuthFilesCooldowns(t, h, "?name=auth-valid-token.json") + + if len(payload.Files) != 1 { + t.Fatalf("expected 1 file, got %d", len(payload.Files)) + } + + file := payload.Files[0] + if file.Unavailable { + t.Error("expected unavailable=false for valid token with scheduled refresh retry, got true") + } + if file.Status != string(coreauth.StatusActive) { + t.Errorf("expected status=%q, got %q", coreauth.StatusActive, file.Status) + } +} + +func TestListAuthFiles_UnexpiredTokenWithRefresh401_ExpiredCooldown(t *testing.T) { + t.Setenv("MANAGEMENT_PASSWORD", "") + now := time.Now().UTC() + futureExpiry := now.Add(48 * time.Hour).Format(time.RFC3339) + expiredCooldown := now.Add(-10 * time.Minute) + retryBackoff := now.Add(5 * time.Minute) + + manager := coreauth.NewManager(nil, nil, nil) + cfg := &config.Config{AuthDir: t.TempDir()} + manager.SetConfig(cfg) + + // An OAuth credential with a valid access token (+48h) that previously entered quota cooldown (now expired), + // and has a pending refresh retry after a 401 refresh error. + registerAuthForLookupTest(t, manager, &coreauth.Auth{ + ID: "auth-valid-token-expired-cooldown", + Index: "idx-valid-token-exp-cool", + FileName: "auth-valid-token-exp-cool.json", + Provider: "codex", + Status: coreauth.StatusError, + StatusMessage: "credential_quota", + Unavailable: true, + NextRetryAfter: expiredCooldown, + NextRefreshAfter: retryBackoff, + LastError: &coreauth.Error{HTTPStatus: 401, Message: "401 unauthorized on refresh"}, + Quota: coreauth.QuotaState{ + Exceeded: true, + Reason: "credential_quota", + NextRecoverAt: expiredCooldown, + }, + Metadata: map[string]any{ + "type": "codex", + "access_token": "valid-future-access-token", + "expired": futureExpiry, + }, + Attributes: map[string]string{"runtime_only": "true"}, + }) + + h := NewHandlerWithoutConfigFilePath(cfg, manager) + payload := requestAuthFilesCooldowns(t, h, "?name=auth-valid-token-exp-cool.json") + + if len(payload.Files) != 1 { + t.Fatalf("expected 1 file, got %d", len(payload.Files)) + } + + file := payload.Files[0] + if file.Unavailable { + t.Error("expected unavailable=false after quota cooldown expires for valid token, got true") + } + if file.Status != string(coreauth.StatusActive) { + t.Errorf("expected status=%q, got %q", coreauth.StatusActive, file.Status) + } + if file.StatusMessage != "" { + t.Errorf("expected empty status_message, got %q", file.StatusMessage) + } + if !file.NextRetryAfter.IsZero() { + t.Errorf("expected zero next_retry_after, got %v", file.NextRetryAfter) + } +} + +func TestListAuthFiles_ModelLevel403CooldownExpired(t *testing.T) { + t.Setenv("MANAGEMENT_PASSWORD", "") + now := time.Now().UTC() + expiredDeadline := now.Add(-10 * time.Minute) + + manager := coreauth.NewManager(nil, nil, nil) + cfg := &config.Config{AuthDir: t.TempDir()} + manager.SetConfig(cfg) + + // An auth whose model failed with 403 (forbidden) and the conductor copied the error to auth-level, + // but the model-level cooldown has now elapsed. + registerAuthForLookupTest(t, manager, &coreauth.Auth{ + ID: "auth-model-403-expired", + Index: "idx-model-403-exp", + FileName: "auth-model-403-exp.json", + Provider: "codex", + Status: coreauth.StatusError, + StatusMessage: "forbidden", + Unavailable: true, + NextRetryAfter: expiredDeadline, + LastError: &coreauth.Error{HTTPStatus: 403, Message: "forbidden"}, + ModelStates: map[string]*coreauth.ModelState{ + "model-a": { + Status: coreauth.StatusError, + StatusMessage: "forbidden", + Unavailable: true, + NextRetryAfter: expiredDeadline, + LastError: &coreauth.Error{HTTPStatus: 403, Message: "forbidden"}, + }, + }, + Attributes: map[string]string{"runtime_only": "true"}, + }) + + h := NewHandlerWithoutConfigFilePath(cfg, manager) + payload := requestAuthFilesCooldowns(t, h, "?name=auth-model-403-exp.json") + + if len(payload.Files) != 1 { + t.Fatalf("expected 1 file, got %d", len(payload.Files)) + } + + file := payload.Files[0] + if file.Unavailable { + t.Error("expected unavailable=false after model-level 403 cooldown expires, got true") + } + if file.Status != string(coreauth.StatusActive) { + t.Errorf("expected status=%q, got %q", coreauth.StatusActive, file.Status) + } + if file.StatusMessage != "" { + t.Errorf("expected empty status_message after cooldown expires, got %q", file.StatusMessage) + } + if !file.NextRetryAfter.IsZero() { + t.Errorf("expected zero next_retry_after, got %v", file.NextRetryAfter) + } +} + +func TestListAuthFiles_SingleModel403Cooling_OtherModelActive(t *testing.T) { + t.Setenv("MANAGEMENT_PASSWORD", "") + now := time.Now().UTC() + activeDeadline := now.Add(30 * time.Minute) + + manager := coreauth.NewManager(nil, nil, nil) + cfg := &config.Config{AuthDir: t.TempDir()} + manager.SetConfig(cfg) + + // An auth where Model A is in a 403 cooldown, but Model B is active and healthy. + registerAuthForLookupTest(t, manager, &coreauth.Auth{ + ID: "auth-single-403-other-active", + Index: "idx-single-403", + FileName: "auth-single-403.json", + Provider: "codex", + Status: coreauth.StatusError, + StatusMessage: "forbidden", + Unavailable: false, + LastError: &coreauth.Error{HTTPStatus: 403, Message: "forbidden"}, + ModelStates: map[string]*coreauth.ModelState{ + "model-forbidden": { + Status: coreauth.StatusError, + StatusMessage: "forbidden", + Unavailable: true, + NextRetryAfter: activeDeadline, + LastError: &coreauth.Error{HTTPStatus: 403, Message: "forbidden"}, + }, + "model-working": { + Status: coreauth.StatusActive, + }, + }, + Attributes: map[string]string{"runtime_only": "true"}, + }) + + h := NewHandlerWithoutConfigFilePath(cfg, manager) + payload := requestAuthFilesCooldowns(t, h, "?name=auth-single-403.json") + + if len(payload.Files) != 1 { + t.Fatalf("expected 1 file, got %d", len(payload.Files)) + } + + file := payload.Files[0] + if file.Unavailable { + t.Error("expected unavailable=false because model-working is active, got true") + } + if file.Status != string(coreauth.StatusActive) { + t.Errorf("expected status=%q because model-working is active, got %q", coreauth.StatusActive, file.Status) + } +} + +func TestListAuthFiles_ExpiredCooldown_ExpiredToken_InFlightRefresh(t *testing.T) { + t.Setenv("MANAGEMENT_PASSWORD", "") + now := time.Now().UTC() + expiredDeadline := now.Add(-10 * time.Minute) + inFlightRefresh := now.Add(1 * time.Minute) + + manager := coreauth.NewManager(nil, nil, nil) + cfg := &config.Config{AuthDir: t.TempDir()} + manager.SetConfig(cfg) + + // An OAuth credential whose quota cooldown expired, but its access token is expired, + // and a background refresh is currently in-flight/scheduled (NextRefreshAfter in future). + // Because the access token is expired, selector blocks it and it must NOT be reported as active. + registerAuthForLookupTest(t, manager, &coreauth.Auth{ + ID: "auth-expired-cooldown-expired-token", + Index: "idx-exp-cool-exp-tok", + FileName: "auth-exp-cool-exp-tok.json", + Provider: "codex", + Status: coreauth.StatusError, + StatusMessage: "credential_quota", + Unavailable: true, + NextRetryAfter: expiredDeadline, + NextRefreshAfter: inFlightRefresh, + Quota: coreauth.QuotaState{ + Exceeded: true, + Reason: "credential_quota", + NextRecoverAt: expiredDeadline, + }, + Metadata: map[string]any{ + "type": "codex", + "access_token": "expired-access-token", + "expired": now.Add(-5 * time.Minute).Format(time.RFC3339), + }, + Attributes: map[string]string{"runtime_only": "true"}, + }) + + h := NewHandlerWithoutConfigFilePath(cfg, manager) + payload := requestAuthFilesCooldowns(t, h, "?name=auth-exp-cool-exp-tok.json") + + if len(payload.Files) != 1 { + t.Fatalf("expected 1 file, got %d", len(payload.Files)) + } + + file := payload.Files[0] + if !file.Unavailable { + t.Error("expected unavailable=true because access token is expired, got false") + } + if file.Status != string(coreauth.StatusError) { + t.Errorf("expected status=%q, got %q", coreauth.StatusError, file.Status) + } + if file.StatusMessage != "credential_quota" { + t.Errorf("expected status_message=%q, got %q", "credential_quota", file.StatusMessage) + } +} + +func TestListAuthFiles_ActiveAuth_InactiveFutureTimestamp(t *testing.T) { + t.Setenv("MANAGEMENT_PASSWORD", "") + now := time.Now().UTC() + inactiveFutureTimestamp := now.Add(1 * time.Hour) + + manager := coreauth.NewManager(nil, nil, nil) + cfg := &config.Config{AuthDir: t.TempDir()} + manager.SetConfig(cfg) + + // An active, healthy auth that has an inactive future NextRetryAfter timestamp + // (Unavailable=false, Quota.Exceeded=false). Matching selector.availabilityBlock, + // an inactive timestamp does not block the credential or report it as in error. + registerAuthForLookupTest(t, manager, &coreauth.Auth{ + ID: "auth-active-inactive-future-timestamp", + Index: "idx-active-inactive-future", + FileName: "auth-active-future.json", + Provider: "codex", + Status: coreauth.StatusActive, + Unavailable: false, + NextRetryAfter: inactiveFutureTimestamp, + Attributes: map[string]string{"runtime_only": "true"}, + }) + + h := NewHandlerWithoutConfigFilePath(cfg, manager) + payload := requestAuthFilesCooldowns(t, h, "?name=auth-active-future.json") + + if len(payload.Files) != 1 { + t.Fatalf("expected 1 file, got %d", len(payload.Files)) + } + + file := payload.Files[0] + if file.Unavailable { + t.Error("expected unavailable=false for active auth with inactive future timestamp, got true") + } + if file.Status != string(coreauth.StatusActive) { + t.Errorf("expected status=%q, got %q", coreauth.StatusActive, file.Status) + } +} + +func TestListAuthFiles_ModelCooling_OtherModelPermanentBlocked(t *testing.T) { + t.Setenv("MANAGEMENT_PASSWORD", "") + now := time.Now().UTC() + activeDeadline := now.Add(30 * time.Minute) + + manager := coreauth.NewManager(nil, nil, nil) + cfg := &config.Config{AuthDir: t.TempDir()} + manager.SetConfig(cfg) + + // Model A is in active cooldown (+30m). + // Model B has a permanent failure with no recovery time (Unavailable=true, NextRetryAfter=zero). + // Because all schedulable models are blocked, the credential as a whole cannot serve any model. + registerAuthForLookupTest(t, manager, &coreauth.Auth{ + ID: "auth-all-blocked-cooling-and-perm", + Index: "idx-all-blocked", + FileName: "auth-all-blocked.json", + Provider: "codex", + Status: coreauth.StatusError, + StatusMessage: "model failure", + Unavailable: true, + ModelStates: map[string]*coreauth.ModelState{ + "model-cooling": { + Status: coreauth.StatusError, + Unavailable: true, + NextRetryAfter: activeDeadline, + }, + "model-perm-blocked": { + Status: coreauth.StatusError, + Unavailable: true, + }, + }, + Attributes: map[string]string{"runtime_only": "true"}, + }) + + h := NewHandlerWithoutConfigFilePath(cfg, manager) + payload := requestAuthFilesCooldowns(t, h, "?name=auth-all-blocked.json") + + if len(payload.Files) != 1 { + t.Fatalf("expected 1 file, got %d", len(payload.Files)) + } + + file := payload.Files[0] + if !file.Unavailable { + t.Error("expected unavailable=true because all models are blocked, got false") + } + if file.Status != string(coreauth.StatusError) { + t.Errorf("expected status=%q because all models are blocked, got %q", coreauth.StatusError, file.Status) + } +} + +func TestListAuthFiles_SparseModelState_BlockedModelDoesNotMakeAuthUnavailable(t *testing.T) { + t.Setenv("MANAGEMENT_PASSWORD", "") + now := time.Now().UTC() + activeDeadline := now.Add(30 * time.Minute) + + manager := coreauth.NewManager(nil, nil, nil) + cfg := &config.Config{AuthDir: t.TempDir()} + manager.SetConfig(cfg) + + // An auth where Model A has an active cooldown in ModelStates, but Model B has no entry in the sparse map yet. + // Because other supported models are schedulable and auth.Unavailable is false, the credential as a whole + // must NOT be reported as unavailable. + registerAuthForLookupTest(t, manager, &coreauth.Auth{ + ID: "auth-sparse-models", + Index: "idx-sparse-models", + FileName: "auth-sparse-models.json", + Provider: "codex", + Status: coreauth.StatusActive, + Unavailable: false, + ModelStates: map[string]*coreauth.ModelState{ + "model-a": { + Status: coreauth.StatusError, + Unavailable: true, + NextRetryAfter: activeDeadline, + }, + }, + Attributes: map[string]string{"runtime_only": "true"}, + }) + + h := NewHandlerWithoutConfigFilePath(cfg, manager) + payload := requestAuthFilesCooldowns(t, h, "?name=auth-sparse-models.json") + + if len(payload.Files) != 1 { + t.Fatalf("expected 1 file, got %d", len(payload.Files)) + } + + file := payload.Files[0] + if file.Unavailable { + t.Error("expected unavailable=false because sparse unrecorded models remain schedulable, got true") + } + if file.Status != string(coreauth.StatusActive) { + t.Errorf("expected status=%q, got %q", coreauth.StatusActive, file.Status) + } +} diff --git a/sdk/cliproxy/auth/conductor_cooldown.go b/sdk/cliproxy/auth/conductor_cooldown.go index 22041fa59..c2a855c25 100644 --- a/sdk/cliproxy/auth/conductor_cooldown.go +++ b/sdk/cliproxy/auth/conductor_cooldown.go @@ -1663,6 +1663,12 @@ func hasUnauthorizedAuthFailure(auth *Auth) bool { return false } +// HasUnauthorizedAuthFailure reports whether the auth has a terminal unauthorized error +// with no pending refresh scheduled. +func HasUnauthorizedAuthFailure(auth *Auth) bool { + return hasUnauthorizedAuthFailure(auth) +} + func refreshErrorFromError(err error) *Error { if err == nil { return nil