fix(management): reconcile expired cooldowns to active status in auth files

- Reconcile expired credential and model cooldown states when building auth file entries
- Preserve error status and unavailability for persistent auth or token expiration failures
- Clear expired retry deadlines and restore active availability when cooldowns elapse

Closes: #5964
This commit is contained in:
Luis Pater
2026-09-19 22:24:40 +08:00
parent b532db9c4b
commit b4ff581daf
3 changed files with 736 additions and 5 deletions

View File

@@ -484,6 +484,156 @@ func (h *Handler) buildAuthFileEntry(auth *coreauth.Auth, quotaSupported ...map[
return h.buildAuthFileEntryLocked(auth, quotaSupported...)
}
func isPersistentAuthFailure(auth *coreauth.Auth, now time.Time) bool {
if auth == nil {
return false
}
// Terminal unauthorized failure with no refresh scheduled.
if coreauth.HasUnauthorizedAuthFailure(auth) {
return true
}
// An OAuth credential whose access token is expired cannot be used to serve requests.
if exp, ok := auth.AccessTokenExpirationTime(); ok && !exp.IsZero() && !exp.After(now) {
return true
}
// An explicit token expiration status.
if strings.EqualFold(strings.TrimSpace(auth.StatusMessage), "token expired") {
return true
}
return false
}
func isModelStateBlocked(state *coreauth.ModelState, now time.Time) bool {
if state == nil {
return false
}
if state.Status == coreauth.StatusDisabled {
return true
}
if !state.Unavailable && !state.Quota.Exceeded {
return false
}
hasRecoveryTime := !state.NextRetryAfter.IsZero() || (!state.Quota.NextRecoverAt.IsZero() && state.Quota.Exceeded)
if !state.NextRetryAfter.IsZero() && state.NextRetryAfter.After(now) {
return true
}
if state.Quota.Exceeded && !state.Quota.NextRecoverAt.IsZero() && state.Quota.NextRecoverAt.After(now) {
return true
}
if hasRecoveryTime {
return false
}
return true
}
func reconcileAuthFileCooldownState(auth *coreauth.Auth, now time.Time) (unavailable bool, status coreauth.Status, statusMessage string, nextRetry time.Time) {
if auth == nil {
return false, coreauth.StatusActive, "", time.Time{}
}
unavailable = auth.Unavailable
status = auth.Status
statusMessage = auth.StatusMessage
if !auth.NextRetryAfter.IsZero() {
nextRetry = auth.NextRetryAfter
}
if auth.Disabled || auth.Status == coreauth.StatusDisabled {
return unavailable, coreauth.StatusDisabled, statusMessage, nextRetry
}
// Never reconcile an active authentication or token failure to active.
if isPersistentAuthFailure(auth, now) {
if !nextRetry.IsZero() && !nextRetry.After(now) {
nextRetry = time.Time{}
}
return true, coreauth.StatusError, statusMessage, nextRetry
}
// Check if there is an active credential-level cooldown.
// Matching selector.availabilityBlock: if neither Unavailable nor Quota.Exceeded is true,
// an inactive timestamp does not block the credential.
hasActiveCredCooldown := false
if auth.Unavailable || auth.Quota.Exceeded {
if !auth.NextRetryAfter.IsZero() && auth.NextRetryAfter.After(now) {
hasActiveCredCooldown = true
}
if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now) {
hasActiveCredCooldown = true
if nextRetry.IsZero() || auth.Quota.NextRecoverAt.After(nextRetry) {
nextRetry = auth.Quota.NextRecoverAt
}
}
}
// Check per-model states.
hasSchedulableModels := false
allSchedulableBlocked := true
hasActiveModelCooldown := false
hadAnyModelCooldown := false
for _, state := range auth.ModelStates {
if state == nil {
continue
}
if state.Status == coreauth.StatusDisabled {
continue
}
hasSchedulableModels = true
if !state.NextRetryAfter.IsZero() || (state.Quota.Exceeded && !state.Quota.NextRecoverAt.IsZero()) {
hadAnyModelCooldown = true
}
if (!state.NextRetryAfter.IsZero() && state.NextRetryAfter.After(now)) ||
(state.Quota.Exceeded && !state.Quota.NextRecoverAt.IsZero() && state.Quota.NextRecoverAt.After(now)) {
hasActiveModelCooldown = true
}
if !isModelStateBlocked(state, now) {
allSchedulableBlocked = false
}
}
hadCooldown := !auth.NextRetryAfter.IsZero() ||
(auth.Quota.Exceeded && !auth.Quota.NextRecoverAt.IsZero()) ||
hadAnyModelCooldown
// If there is an active credential cooldown, keep unavailable/error.
// If all recorded models are blocked and the credential itself was marked unavailable, keep unavailable/error.
if hasActiveCredCooldown || (hasSchedulableModels && allSchedulableBlocked && auth.Unavailable) {
if !nextRetry.IsZero() && !nextRetry.After(now) {
nextRetry = time.Time{}
}
return true, coreauth.StatusError, statusMessage, nextRetry
}
// If the credential was not marked unavailable and has no active credential cooldown, keep unavailable=false.
if !auth.Unavailable && !hasActiveCredCooldown {
if status == coreauth.StatusError && hasSchedulableModels && !allSchedulableBlocked {
status = coreauth.StatusActive
statusMessage = ""
}
return false, status, statusMessage, time.Time{}
}
// If a cooldown was recorded but has expired (and no active model cooldown blocks all models):
if hadCooldown && !hasActiveCredCooldown && !hasActiveModelCooldown {
return false, coreauth.StatusActive, "", time.Time{}
}
// If partial models are still cooling, the credential as a whole remains available for other models.
if hadCooldown && hasSchedulableModels && !allSchedulableBlocked {
if status == coreauth.StatusError && !hasActiveCredCooldown {
status = coreauth.StatusActive
statusMessage = ""
}
return false, status, statusMessage, time.Time{}
}
// If nextRetry is in the past, do not expose a past retry deadline.
if !nextRetry.IsZero() && !nextRetry.After(now) {
nextRetry = time.Time{}
}
return unavailable, status, statusMessage, nextRetry
}
func (h *Handler) buildAuthFileEntryLocked(auth *coreauth.Auth, quotaSupported ...map[string]struct{}) gin.H {
if auth == nil {
return nil
@@ -501,6 +651,7 @@ func (h *Handler) buildAuthFileEntryLocked(auth *coreauth.Auth, quotaSupported .
if name == "" {
name = auth.ID
}
unavailable, status, statusMessage, nextRetryAfter := reconcileAuthFileCooldownState(auth, time.Now().UTC())
entry := gin.H{
"id": auth.ID,
"auth_index": auth.Index,
@@ -508,10 +659,10 @@ func (h *Handler) buildAuthFileEntryLocked(auth *coreauth.Auth, quotaSupported .
"type": strings.TrimSpace(auth.Provider),
"provider": strings.TrimSpace(auth.Provider),
"label": auth.Label,
"status": auth.Status,
"status_message": auth.StatusMessage,
"status": status,
"status_message": statusMessage,
"disabled": auth.Disabled,
"unavailable": auth.Unavailable,
"unavailable": unavailable,
"runtime_only": runtimeOnly,
"source": "memory",
"size": int64(0),
@@ -571,8 +722,8 @@ func (h *Handler) buildAuthFileEntryLocked(auth *coreauth.Auth, quotaSupported .
if !auth.LastRefreshedAt.IsZero() {
entry["last_refresh"] = auth.LastRefreshedAt
}
if !auth.NextRetryAfter.IsZero() {
entry["next_retry_after"] = auth.NextRetryAfter
if !nextRetryAfter.IsZero() {
entry["next_retry_after"] = nextRetryAfter
}
if path != "" {
entry["path"] = path

View File

@@ -23,6 +23,7 @@ type authFilesCooldownResponse struct {
AuthIndex string `json:"auth_index"`
Name string `json:"name"`
Status string `json:"status"`
StatusMessage string `json:"status_message"`
Unavailable bool `json:"unavailable"`
NextRetryAfter time.Time `json:"next_retry_after"`
Cooldowns json.RawMessage `json:"cooldowns"`
@@ -187,3 +188,576 @@ func TestListAuthFilesCooldownsUnknown(t *testing.T) {
})
}
}
func TestListAuthFiles_ExpiredCooldownReconciledToActive(t *testing.T) {
t.Setenv("MANAGEMENT_PASSWORD", "")
now := time.Now().UTC()
expiredDeadline := now.Add(-10 * time.Minute)
manager := coreauth.NewManager(nil, nil, nil)
cfg := &config.Config{AuthDir: t.TempDir()}
manager.SetConfig(cfg)
// Credential-level expired cooldown (e.g. Codex quota exhaustion in Issue 5964)
registerAuthForLookupTest(t, manager, &coreauth.Auth{
ID: "codex-expired",
Index: "idx-codex-expired",
FileName: "codex-expired.json",
Provider: "codex",
Status: coreauth.StatusError,
StatusMessage: "credential_quota",
Unavailable: true,
NextRetryAfter: expiredDeadline,
Quota: coreauth.QuotaState{
Exceeded: true,
Reason: "credential_quota",
NextRecoverAt: expiredDeadline,
ObservedAt: expiredDeadline,
},
Attributes: map[string]string{"runtime_only": "true"},
})
// Model-level expired cooldown where all model cooldowns have elapsed
registerAuthForLookupTest(t, manager, &coreauth.Auth{
ID: "model-expired",
Index: "idx-model-expired",
FileName: "model-expired.json",
Provider: "claude",
Status: coreauth.StatusError,
StatusMessage: "rate limit exceeded",
Unavailable: true,
NextRetryAfter: expiredDeadline,
ModelStates: map[string]*coreauth.ModelState{
"claude-3-5-sonnet": {
Status: coreauth.StatusError,
StatusMessage: "rate limit exceeded",
Unavailable: true,
NextRetryAfter: expiredDeadline,
},
},
Attributes: map[string]string{"runtime_only": "true"},
})
h := NewHandlerWithoutConfigFilePath(cfg, manager)
payload := requestAuthFilesCooldowns(t, h, "")
if len(payload.Files) != 2 {
t.Fatalf("expected 2 files, got %d", len(payload.Files))
}
for _, file := range payload.Files {
if file.Unavailable {
t.Errorf("auth %s: expected unavailable=false after cooldown expiration, got true", file.ID)
}
if file.Status != string(coreauth.StatusActive) {
t.Errorf("auth %s: expected status=%q after cooldown expiration, got %q", file.ID, coreauth.StatusActive, file.Status)
}
if file.StatusMessage != "" {
t.Errorf("auth %s: expected empty status_message after cooldown expiration, got %q", file.ID, file.StatusMessage)
}
if !file.NextRetryAfter.IsZero() {
t.Errorf("auth %s: expected zero next_retry_after after cooldown expiration, got %v", file.ID, file.NextRetryAfter)
}
if string(file.Cooldowns) != "[]" {
t.Errorf("auth %s: expected cooldowns=[], got %s", file.ID, file.Cooldowns)
}
}
}
func TestListAuthFiles_ExpiredCooldownWithSubsequentTokenFailure(t *testing.T) {
t.Setenv("MANAGEMENT_PASSWORD", "")
now := time.Now().UTC()
expiredDeadline := now.Add(-10 * time.Minute)
manager := coreauth.NewManager(nil, nil, nil)
cfg := &config.Config{AuthDir: t.TempDir()}
manager.SetConfig(cfg)
// An OAuth auth that had a cooldown, but subsequently had its access token expire / refresh fail.
registerAuthForLookupTest(t, manager, &coreauth.Auth{
ID: "codex-token-expired",
Index: "idx-codex-token-expired",
FileName: "codex-token-expired.json",
Provider: "codex",
Status: coreauth.StatusError,
StatusMessage: "token expired",
Unavailable: true,
NextRetryAfter: expiredDeadline,
Metadata: map[string]any{
"type": "codex",
"access_token": "expired-access-token",
"expired": now.Add(-5 * time.Minute).Format(time.RFC3339),
},
Attributes: map[string]string{"runtime_only": "true"},
})
h := NewHandlerWithoutConfigFilePath(cfg, manager)
payload := requestAuthFilesCooldowns(t, h, "?name=codex-token-expired.json")
if len(payload.Files) != 1 {
t.Fatalf("expected 1 file, got %d", len(payload.Files))
}
file := payload.Files[0]
if !file.Unavailable {
t.Error("expected unavailable=true for expired token failure, got false")
}
if file.Status != string(coreauth.StatusError) {
t.Errorf("expected status=%q for expired token failure, got %q", coreauth.StatusError, file.Status)
}
if file.StatusMessage != "token expired" {
t.Errorf("expected status_message=%q, got %q", "token expired", file.StatusMessage)
}
if !file.NextRetryAfter.IsZero() {
t.Errorf("expected zero next_retry_after, got %v", file.NextRetryAfter)
}
}
func TestListAuthFiles_PartialModelCooldownWithAuthFailure(t *testing.T) {
t.Setenv("MANAGEMENT_PASSWORD", "")
now := time.Now().UTC()
activeDeadline := now.Add(10 * time.Minute)
manager := coreauth.NewManager(nil, nil, nil)
cfg := &config.Config{AuthDir: t.TempDir()}
manager.SetConfig(cfg)
// Auth has an independent auth error ("unauthorized"), but only one of its models is in cooldown.
registerAuthForLookupTest(t, manager, &coreauth.Auth{
ID: "auth-unauthorized-partial-model",
Index: "idx-auth-unauthorized",
FileName: "auth-unauthorized.json",
Provider: "claude",
Status: coreauth.StatusError,
StatusMessage: "unauthorized",
Unavailable: true,
LastError: &coreauth.Error{HTTPStatus: 401, Message: "unauthorized"},
ModelStates: map[string]*coreauth.ModelState{
"model-cool": {
Status: coreauth.StatusError,
Unavailable: true,
NextRetryAfter: activeDeadline,
},
"model-free": {
Status: coreauth.StatusActive,
},
},
Attributes: map[string]string{"runtime_only": "true"},
})
h := NewHandlerWithoutConfigFilePath(cfg, manager)
payload := requestAuthFilesCooldowns(t, h, "?name=auth-unauthorized.json")
if len(payload.Files) != 1 {
t.Fatalf("expected 1 file, got %d", len(payload.Files))
}
file := payload.Files[0]
if !file.Unavailable {
t.Error("expected unavailable=true due to unauthorized error, got false")
}
if file.Status != string(coreauth.StatusError) {
t.Errorf("expected status=%q, got %q", coreauth.StatusError, file.Status)
}
if file.StatusMessage != "unauthorized" {
t.Errorf("expected status_message=%q, got %q", "unauthorized", file.StatusMessage)
}
}
func TestListAuthFiles_UnexpiredTokenWithRefresh401_NoCooldown(t *testing.T) {
t.Setenv("MANAGEMENT_PASSWORD", "")
now := time.Now().UTC()
futureExpiry := now.Add(48 * time.Hour).Format(time.RFC3339)
retryBackoff := now.Add(5 * time.Minute)
manager := coreauth.NewManager(nil, nil, nil)
cfg := &config.Config{AuthDir: t.TempDir()}
manager.SetConfig(cfg)
// An OAuth credential whose access token is still valid (+48h), but background refresh failed with 401
// and scheduled a retry backoff. The credential itself is still active and usable.
registerAuthForLookupTest(t, manager, &coreauth.Auth{
ID: "auth-valid-token-refresh-401",
Index: "idx-valid-token",
FileName: "auth-valid-token.json",
Provider: "codex",
Status: coreauth.StatusActive,
Unavailable: false,
NextRefreshAfter: retryBackoff,
LastError: &coreauth.Error{HTTPStatus: 401, Message: "401 unauthorized on refresh"},
Metadata: map[string]any{
"type": "codex",
"access_token": "valid-future-access-token",
"expired": futureExpiry,
},
Attributes: map[string]string{"runtime_only": "true"},
})
h := NewHandlerWithoutConfigFilePath(cfg, manager)
payload := requestAuthFilesCooldowns(t, h, "?name=auth-valid-token.json")
if len(payload.Files) != 1 {
t.Fatalf("expected 1 file, got %d", len(payload.Files))
}
file := payload.Files[0]
if file.Unavailable {
t.Error("expected unavailable=false for valid token with scheduled refresh retry, got true")
}
if file.Status != string(coreauth.StatusActive) {
t.Errorf("expected status=%q, got %q", coreauth.StatusActive, file.Status)
}
}
func TestListAuthFiles_UnexpiredTokenWithRefresh401_ExpiredCooldown(t *testing.T) {
t.Setenv("MANAGEMENT_PASSWORD", "")
now := time.Now().UTC()
futureExpiry := now.Add(48 * time.Hour).Format(time.RFC3339)
expiredCooldown := now.Add(-10 * time.Minute)
retryBackoff := now.Add(5 * time.Minute)
manager := coreauth.NewManager(nil, nil, nil)
cfg := &config.Config{AuthDir: t.TempDir()}
manager.SetConfig(cfg)
// An OAuth credential with a valid access token (+48h) that previously entered quota cooldown (now expired),
// and has a pending refresh retry after a 401 refresh error.
registerAuthForLookupTest(t, manager, &coreauth.Auth{
ID: "auth-valid-token-expired-cooldown",
Index: "idx-valid-token-exp-cool",
FileName: "auth-valid-token-exp-cool.json",
Provider: "codex",
Status: coreauth.StatusError,
StatusMessage: "credential_quota",
Unavailable: true,
NextRetryAfter: expiredCooldown,
NextRefreshAfter: retryBackoff,
LastError: &coreauth.Error{HTTPStatus: 401, Message: "401 unauthorized on refresh"},
Quota: coreauth.QuotaState{
Exceeded: true,
Reason: "credential_quota",
NextRecoverAt: expiredCooldown,
},
Metadata: map[string]any{
"type": "codex",
"access_token": "valid-future-access-token",
"expired": futureExpiry,
},
Attributes: map[string]string{"runtime_only": "true"},
})
h := NewHandlerWithoutConfigFilePath(cfg, manager)
payload := requestAuthFilesCooldowns(t, h, "?name=auth-valid-token-exp-cool.json")
if len(payload.Files) != 1 {
t.Fatalf("expected 1 file, got %d", len(payload.Files))
}
file := payload.Files[0]
if file.Unavailable {
t.Error("expected unavailable=false after quota cooldown expires for valid token, got true")
}
if file.Status != string(coreauth.StatusActive) {
t.Errorf("expected status=%q, got %q", coreauth.StatusActive, file.Status)
}
if file.StatusMessage != "" {
t.Errorf("expected empty status_message, got %q", file.StatusMessage)
}
if !file.NextRetryAfter.IsZero() {
t.Errorf("expected zero next_retry_after, got %v", file.NextRetryAfter)
}
}
func TestListAuthFiles_ModelLevel403CooldownExpired(t *testing.T) {
t.Setenv("MANAGEMENT_PASSWORD", "")
now := time.Now().UTC()
expiredDeadline := now.Add(-10 * time.Minute)
manager := coreauth.NewManager(nil, nil, nil)
cfg := &config.Config{AuthDir: t.TempDir()}
manager.SetConfig(cfg)
// An auth whose model failed with 403 (forbidden) and the conductor copied the error to auth-level,
// but the model-level cooldown has now elapsed.
registerAuthForLookupTest(t, manager, &coreauth.Auth{
ID: "auth-model-403-expired",
Index: "idx-model-403-exp",
FileName: "auth-model-403-exp.json",
Provider: "codex",
Status: coreauth.StatusError,
StatusMessage: "forbidden",
Unavailable: true,
NextRetryAfter: expiredDeadline,
LastError: &coreauth.Error{HTTPStatus: 403, Message: "forbidden"},
ModelStates: map[string]*coreauth.ModelState{
"model-a": {
Status: coreauth.StatusError,
StatusMessage: "forbidden",
Unavailable: true,
NextRetryAfter: expiredDeadline,
LastError: &coreauth.Error{HTTPStatus: 403, Message: "forbidden"},
},
},
Attributes: map[string]string{"runtime_only": "true"},
})
h := NewHandlerWithoutConfigFilePath(cfg, manager)
payload := requestAuthFilesCooldowns(t, h, "?name=auth-model-403-exp.json")
if len(payload.Files) != 1 {
t.Fatalf("expected 1 file, got %d", len(payload.Files))
}
file := payload.Files[0]
if file.Unavailable {
t.Error("expected unavailable=false after model-level 403 cooldown expires, got true")
}
if file.Status != string(coreauth.StatusActive) {
t.Errorf("expected status=%q, got %q", coreauth.StatusActive, file.Status)
}
if file.StatusMessage != "" {
t.Errorf("expected empty status_message after cooldown expires, got %q", file.StatusMessage)
}
if !file.NextRetryAfter.IsZero() {
t.Errorf("expected zero next_retry_after, got %v", file.NextRetryAfter)
}
}
func TestListAuthFiles_SingleModel403Cooling_OtherModelActive(t *testing.T) {
t.Setenv("MANAGEMENT_PASSWORD", "")
now := time.Now().UTC()
activeDeadline := now.Add(30 * time.Minute)
manager := coreauth.NewManager(nil, nil, nil)
cfg := &config.Config{AuthDir: t.TempDir()}
manager.SetConfig(cfg)
// An auth where Model A is in a 403 cooldown, but Model B is active and healthy.
registerAuthForLookupTest(t, manager, &coreauth.Auth{
ID: "auth-single-403-other-active",
Index: "idx-single-403",
FileName: "auth-single-403.json",
Provider: "codex",
Status: coreauth.StatusError,
StatusMessage: "forbidden",
Unavailable: false,
LastError: &coreauth.Error{HTTPStatus: 403, Message: "forbidden"},
ModelStates: map[string]*coreauth.ModelState{
"model-forbidden": {
Status: coreauth.StatusError,
StatusMessage: "forbidden",
Unavailable: true,
NextRetryAfter: activeDeadline,
LastError: &coreauth.Error{HTTPStatus: 403, Message: "forbidden"},
},
"model-working": {
Status: coreauth.StatusActive,
},
},
Attributes: map[string]string{"runtime_only": "true"},
})
h := NewHandlerWithoutConfigFilePath(cfg, manager)
payload := requestAuthFilesCooldowns(t, h, "?name=auth-single-403.json")
if len(payload.Files) != 1 {
t.Fatalf("expected 1 file, got %d", len(payload.Files))
}
file := payload.Files[0]
if file.Unavailable {
t.Error("expected unavailable=false because model-working is active, got true")
}
if file.Status != string(coreauth.StatusActive) {
t.Errorf("expected status=%q because model-working is active, got %q", coreauth.StatusActive, file.Status)
}
}
func TestListAuthFiles_ExpiredCooldown_ExpiredToken_InFlightRefresh(t *testing.T) {
t.Setenv("MANAGEMENT_PASSWORD", "")
now := time.Now().UTC()
expiredDeadline := now.Add(-10 * time.Minute)
inFlightRefresh := now.Add(1 * time.Minute)
manager := coreauth.NewManager(nil, nil, nil)
cfg := &config.Config{AuthDir: t.TempDir()}
manager.SetConfig(cfg)
// An OAuth credential whose quota cooldown expired, but its access token is expired,
// and a background refresh is currently in-flight/scheduled (NextRefreshAfter in future).
// Because the access token is expired, selector blocks it and it must NOT be reported as active.
registerAuthForLookupTest(t, manager, &coreauth.Auth{
ID: "auth-expired-cooldown-expired-token",
Index: "idx-exp-cool-exp-tok",
FileName: "auth-exp-cool-exp-tok.json",
Provider: "codex",
Status: coreauth.StatusError,
StatusMessage: "credential_quota",
Unavailable: true,
NextRetryAfter: expiredDeadline,
NextRefreshAfter: inFlightRefresh,
Quota: coreauth.QuotaState{
Exceeded: true,
Reason: "credential_quota",
NextRecoverAt: expiredDeadline,
},
Metadata: map[string]any{
"type": "codex",
"access_token": "expired-access-token",
"expired": now.Add(-5 * time.Minute).Format(time.RFC3339),
},
Attributes: map[string]string{"runtime_only": "true"},
})
h := NewHandlerWithoutConfigFilePath(cfg, manager)
payload := requestAuthFilesCooldowns(t, h, "?name=auth-exp-cool-exp-tok.json")
if len(payload.Files) != 1 {
t.Fatalf("expected 1 file, got %d", len(payload.Files))
}
file := payload.Files[0]
if !file.Unavailable {
t.Error("expected unavailable=true because access token is expired, got false")
}
if file.Status != string(coreauth.StatusError) {
t.Errorf("expected status=%q, got %q", coreauth.StatusError, file.Status)
}
if file.StatusMessage != "credential_quota" {
t.Errorf("expected status_message=%q, got %q", "credential_quota", file.StatusMessage)
}
}
func TestListAuthFiles_ActiveAuth_InactiveFutureTimestamp(t *testing.T) {
t.Setenv("MANAGEMENT_PASSWORD", "")
now := time.Now().UTC()
inactiveFutureTimestamp := now.Add(1 * time.Hour)
manager := coreauth.NewManager(nil, nil, nil)
cfg := &config.Config{AuthDir: t.TempDir()}
manager.SetConfig(cfg)
// An active, healthy auth that has an inactive future NextRetryAfter timestamp
// (Unavailable=false, Quota.Exceeded=false). Matching selector.availabilityBlock,
// an inactive timestamp does not block the credential or report it as in error.
registerAuthForLookupTest(t, manager, &coreauth.Auth{
ID: "auth-active-inactive-future-timestamp",
Index: "idx-active-inactive-future",
FileName: "auth-active-future.json",
Provider: "codex",
Status: coreauth.StatusActive,
Unavailable: false,
NextRetryAfter: inactiveFutureTimestamp,
Attributes: map[string]string{"runtime_only": "true"},
})
h := NewHandlerWithoutConfigFilePath(cfg, manager)
payload := requestAuthFilesCooldowns(t, h, "?name=auth-active-future.json")
if len(payload.Files) != 1 {
t.Fatalf("expected 1 file, got %d", len(payload.Files))
}
file := payload.Files[0]
if file.Unavailable {
t.Error("expected unavailable=false for active auth with inactive future timestamp, got true")
}
if file.Status != string(coreauth.StatusActive) {
t.Errorf("expected status=%q, got %q", coreauth.StatusActive, file.Status)
}
}
func TestListAuthFiles_ModelCooling_OtherModelPermanentBlocked(t *testing.T) {
t.Setenv("MANAGEMENT_PASSWORD", "")
now := time.Now().UTC()
activeDeadline := now.Add(30 * time.Minute)
manager := coreauth.NewManager(nil, nil, nil)
cfg := &config.Config{AuthDir: t.TempDir()}
manager.SetConfig(cfg)
// Model A is in active cooldown (+30m).
// Model B has a permanent failure with no recovery time (Unavailable=true, NextRetryAfter=zero).
// Because all schedulable models are blocked, the credential as a whole cannot serve any model.
registerAuthForLookupTest(t, manager, &coreauth.Auth{
ID: "auth-all-blocked-cooling-and-perm",
Index: "idx-all-blocked",
FileName: "auth-all-blocked.json",
Provider: "codex",
Status: coreauth.StatusError,
StatusMessage: "model failure",
Unavailable: true,
ModelStates: map[string]*coreauth.ModelState{
"model-cooling": {
Status: coreauth.StatusError,
Unavailable: true,
NextRetryAfter: activeDeadline,
},
"model-perm-blocked": {
Status: coreauth.StatusError,
Unavailable: true,
},
},
Attributes: map[string]string{"runtime_only": "true"},
})
h := NewHandlerWithoutConfigFilePath(cfg, manager)
payload := requestAuthFilesCooldowns(t, h, "?name=auth-all-blocked.json")
if len(payload.Files) != 1 {
t.Fatalf("expected 1 file, got %d", len(payload.Files))
}
file := payload.Files[0]
if !file.Unavailable {
t.Error("expected unavailable=true because all models are blocked, got false")
}
if file.Status != string(coreauth.StatusError) {
t.Errorf("expected status=%q because all models are blocked, got %q", coreauth.StatusError, file.Status)
}
}
func TestListAuthFiles_SparseModelState_BlockedModelDoesNotMakeAuthUnavailable(t *testing.T) {
t.Setenv("MANAGEMENT_PASSWORD", "")
now := time.Now().UTC()
activeDeadline := now.Add(30 * time.Minute)
manager := coreauth.NewManager(nil, nil, nil)
cfg := &config.Config{AuthDir: t.TempDir()}
manager.SetConfig(cfg)
// An auth where Model A has an active cooldown in ModelStates, but Model B has no entry in the sparse map yet.
// Because other supported models are schedulable and auth.Unavailable is false, the credential as a whole
// must NOT be reported as unavailable.
registerAuthForLookupTest(t, manager, &coreauth.Auth{
ID: "auth-sparse-models",
Index: "idx-sparse-models",
FileName: "auth-sparse-models.json",
Provider: "codex",
Status: coreauth.StatusActive,
Unavailable: false,
ModelStates: map[string]*coreauth.ModelState{
"model-a": {
Status: coreauth.StatusError,
Unavailable: true,
NextRetryAfter: activeDeadline,
},
},
Attributes: map[string]string{"runtime_only": "true"},
})
h := NewHandlerWithoutConfigFilePath(cfg, manager)
payload := requestAuthFilesCooldowns(t, h, "?name=auth-sparse-models.json")
if len(payload.Files) != 1 {
t.Fatalf("expected 1 file, got %d", len(payload.Files))
}
file := payload.Files[0]
if file.Unavailable {
t.Error("expected unavailable=false because sparse unrecorded models remain schedulable, got true")
}
if file.Status != string(coreauth.StatusActive) {
t.Errorf("expected status=%q, got %q", coreauth.StatusActive, file.Status)
}
}

View File

@@ -1663,6 +1663,12 @@ func hasUnauthorizedAuthFailure(auth *Auth) bool {
return false
}
// HasUnauthorizedAuthFailure reports whether the auth has a terminal unauthorized error
// with no pending refresh scheduled.
func HasUnauthorizedAuthFailure(auth *Auth) bool {
return hasUnauthorizedAuthFailure(auth)
}
func refreshErrorFromError(err error) *Error {
if err == nil {
return nil