From 1ca975dfc011c320fd045a7ce9070d66e418fa22 Mon Sep 17 00:00:00 2001 From: Supra4E8C Date: Sun, 13 Sep 2026 18:50:21 +0800 Subject: [PATCH] feat(cooldowns): add cooldown snapshot feature for management auth files --- .../api/handlers/management/auth_files.go | 15 +- .../management/auth_files_cooldown_test.go | 189 ++++++++++++++ sdk/cliproxy/auth/cooldown_view.go | 171 +++++++++++++ sdk/cliproxy/auth/cooldown_view_test.go | 232 ++++++++++++++++++ 4 files changed, 603 insertions(+), 4 deletions(-) create mode 100644 internal/api/handlers/management/auth_files_cooldown_test.go create mode 100644 sdk/cliproxy/auth/cooldown_view.go create mode 100644 sdk/cliproxy/auth/cooldown_view_test.go diff --git a/internal/api/handlers/management/auth_files.go b/internal/api/handlers/management/auth_files.go index f5695337f..baef9bb81 100644 --- a/internal/api/handlers/management/auth_files.go +++ b/internal/api/handlers/management/auth_files.go @@ -106,12 +106,18 @@ func (h *Handler) ListAuthFiles(c *gin.Context) { quotaSupportedProviders = host.QuotaSupportedProvidersSet(c.Request.Context()) } auths := h.authManager.List() + observedAt := time.Now().UTC() + cooldownsKnown := !h.authManager.HomeEnabled() files := make([]gin.H, 0, len(auths)) for _, auth := range auths { if !matchesAuthFileLookup(auth, nameFilter, authIndexFilter) { continue } if entry := h.buildAuthFileEntry(auth, quotaSupportedProviders); entry != nil { + entry["cooldowns"] = nil + if cooldownsKnown { + entry["cooldowns"] = coreauth.CooldownSnapshotForAuth(auth, observedAt) + } files = append(files, entry) } } @@ -120,7 +126,7 @@ func (h *Handler) ListAuthFiles(c *gin.Context) { nameJ, _ := files[j]["name"].(string) return strings.ToLower(nameI) < strings.ToLower(nameJ) }) - c.JSON(200, gin.H{"files": files}) + c.JSON(200, gin.H{"observed_at": observedAt, "files": files}) } func lockedAuthIndex(auth *coreauth.Auth) string { @@ -222,6 +228,7 @@ func (h *Handler) GetAuthFileModels(c *gin.Context) { // List auth files from disk when the auth manager is unavailable. func (h *Handler) listAuthFilesFromDisk(c *gin.Context) { + observedAt := time.Now().UTC() nameFilter := strings.TrimSpace(c.Query("name")) authIndexFilter := strings.TrimSpace(c.Query("auth_index")) entries, err := os.ReadDir(h.cfg.AuthDir) @@ -231,7 +238,7 @@ func (h *Handler) listAuthFilesFromDisk(c *gin.Context) { } files := make([]gin.H, 0) if authIndexFilter != "" { - c.JSON(200, gin.H{"files": files}) + c.JSON(200, gin.H{"observed_at": observedAt, "files": files}) return } for _, e := range entries { @@ -246,7 +253,7 @@ func (h *Handler) listAuthFilesFromDisk(c *gin.Context) { continue } if info, errInfo := e.Info(); errInfo == nil { - fileData := gin.H{"name": name, "size": info.Size(), "modtime": info.ModTime()} + fileData := gin.H{"name": name, "size": info.Size(), "modtime": info.ModTime(), "cooldowns": nil} // Read file to get type field full := filepath.Join(h.cfg.AuthDir, name) @@ -307,7 +314,7 @@ func (h *Handler) listAuthFilesFromDisk(c *gin.Context) { files = append(files, fileData) } } - c.JSON(200, gin.H{"files": files}) + c.JSON(200, gin.H{"observed_at": observedAt, "files": files}) } func (h *Handler) buildAuthFileEntry(auth *coreauth.Auth, quotaSupported ...map[string]struct{}) gin.H { diff --git a/internal/api/handlers/management/auth_files_cooldown_test.go b/internal/api/handlers/management/auth_files_cooldown_test.go new file mode 100644 index 000000000..a601c5b49 --- /dev/null +++ b/internal/api/handlers/management/auth_files_cooldown_test.go @@ -0,0 +1,189 @@ +package management + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "reflect" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/router-for-me/CLIProxyAPI/v7/internal/config" + coreauth "github.com/router-for-me/CLIProxyAPI/v7/sdk/cliproxy/auth" +) + +type authFilesCooldownResponse struct { + ObservedAt time.Time `json:"observed_at"` + Files []struct { + ID string `json:"id"` + AuthIndex string `json:"auth_index"` + Name string `json:"name"` + Status string `json:"status"` + Unavailable bool `json:"unavailable"` + NextRetryAfter time.Time `json:"next_retry_after"` + Cooldowns json.RawMessage `json:"cooldowns"` + Quota map[string]any `json:"quota"` + ModelQuotas map[string]map[string]any `json:"model_quotas"` + } `json:"files"` +} + +func requestAuthFilesCooldowns(t *testing.T, h *Handler, query string) authFilesCooldownResponse { + t.Helper() + rec := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rec) + ctx.Request = httptest.NewRequest(http.MethodGet, "/v0/management/auth-files"+query, nil) + h.ListAuthFiles(ctx) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d: %s", rec.Code, rec.Body.String()) + } + var payload authFilesCooldownResponse + if errDecode := json.Unmarshal(rec.Body.Bytes(), &payload); errDecode != nil { + t.Fatal(errDecode) + } + if payload.ObservedAt.IsZero() || payload.ObservedAt.Location() != time.UTC { + t.Fatalf("invalid observed_at: %v", payload.ObservedAt) + } + return payload +} + +func TestListAuthFilesCooldownsSnapshot(t *testing.T) { + t.Setenv("MANAGEMENT_PASSWORD", "") + now := time.Now().UTC() + next := now.Add(time.Hour) + manager := coreauth.NewManager(nil, nil, nil) + cfg := &config.Config{AuthDir: t.TempDir()} + manager.SetConfig(cfg) + for _, id := range []string{"a", "b"} { + registerAuthForLookupTest(t, manager, &coreauth.Auth{ + ID: id, Index: "index-" + id, Provider: "codex", Status: coreauth.StatusError, + Unavailable: true, NextRetryAfter: next, + Attributes: map[string]string{"runtime_only": "true"}, + Quota: coreauth.QuotaState{Exceeded: true, Reason: "quota", NextRecoverAt: next, ObservedAt: now, Signals: map[string]string{"x-codex-primary-used-percent": "90"}}, + ModelStates: map[string]*coreauth.ModelState{ + "model-a": {Unavailable: true, NextRetryAfter: next, Quota: coreauth.QuotaState{Exceeded: true, Reason: "quota", NextRecoverAt: next, BackoffLevel: 6, ObservedAt: now, Signals: map[string]string{"x-codex-primary-used-percent": "90"}}, LastError: &coreauth.Error{HTTPStatus: 429, Message: "private upstream body"}}, + "expired": {Status: coreauth.StatusError, Unavailable: true, NextRetryAfter: now.Add(-time.Hour), Quota: coreauth.QuotaState{Exceeded: true, NextRecoverAt: now.Add(-time.Hour), BackoffLevel: 9}}, + }, + }) + } + beforeA, _ := manager.GetByID("a") + beforeB, _ := manager.GetByID("b") + h := NewHandlerWithoutConfigFilePath(cfg, manager) + for range 2 { + payload := requestAuthFilesCooldowns(t, h, "") + if len(payload.Files) != 2 { + t.Fatalf("files = %+v", payload.Files) + } + for i, file := range payload.Files { + if file.ID != []string{"a", "b"}[i] || file.AuthIndex != "index-"+file.ID { + t.Fatalf("identity/order changed: %+v", file) + } + if file.Status != string(coreauth.StatusError) || !file.Unavailable || !file.NextRetryAfter.Equal(next) { + t.Fatalf("existing state changed: %+v", file) + } + var views []coreauth.CooldownView + if errDecode := json.Unmarshal(file.Cooldowns, &views); errDecode != nil { + t.Fatal(errDecode) + } + if len(views) != 1 || views[0].Scope != "model" || views[0].ModelKey != "model-a" || views[0].Reason != "quota" || views[0].HTTPStatus != 429 || views[0].BackoffLevel == nil || *views[0].BackoffLevel != 6 { + t.Fatalf("unexpected cooldowns: %s", file.Cooldowns) + } + remaining := next.Sub(payload.ObservedAt) + wantSeconds := int64(remaining / time.Second) + if remaining%time.Second != 0 { + wantSeconds++ + } + if views[0].RemainingSeconds != wantSeconds || !views[0].RetryAt.Equal(next) { + t.Fatalf("inconsistent time basis: %+v", views[0]) + } + for _, quota := range []map[string]any{file.Quota, file.ModelQuotas["model-a"]} { + if len(quota) != 2 || quota["signals"] == nil || quota["observed_at"] == nil { + t.Fatalf("quota observation changed: %+v", quota) + } + } + var fields []map[string]any + if errDecode := json.Unmarshal(file.Cooldowns, &fields); errDecode != nil { + t.Fatal(errDecode) + } + if len(fields[0]) != 7 { + t.Fatalf("unexpected fields: %+v", fields[0]) + } + } + } + afterA, _ := manager.GetByID("a") + afterB, _ := manager.GetByID("b") + if !reflect.DeepEqual(beforeA, afterA) || !reflect.DeepEqual(beforeB, afterB) { + t.Fatal("GET mutated auth state") + } +} + +func TestListAuthFilesCooldownsCredentialKindsAndFilters(t *testing.T) { + t.Setenv("MANAGEMENT_PASSWORD", "") + cfg := &config.Config{AuthDir: t.TempDir()} + manager := coreauth.NewManager(nil, nil, nil) + manager.SetConfig(cfg) + path := filepath.Join(cfg.AuthDir, "shared.json") + if errWrite := os.WriteFile(path, []byte(`{"type":"codex"}`), 0o600); errWrite != nil { + t.Fatal(errWrite) + } + for _, id := range []string{"file", "virtual", "runtime"} { + auth := &coreauth.Auth{ID: id, Index: "index-" + id, FileName: "shared.json", Provider: "codex", Status: coreauth.StatusActive, Attributes: map[string]string{"path": path}} + if id == "virtual" { + coreauth.MarkPluginVirtualAuth(auth, path, 0) + } + if id == "runtime" { + auth.Attributes = map[string]string{"runtime_only": "true"} + } + registerAuthForLookupTest(t, manager, auth) + } + h := NewHandlerWithoutConfigFilePath(cfg, manager) + payload := requestAuthFilesCooldowns(t, h, "?name=shared.json") + if len(payload.Files) != 3 { + t.Fatalf("files = %+v", payload.Files) + } + for _, file := range payload.Files { + if string(file.Cooldowns) != "[]" { + t.Fatalf("known empty cooldowns = %s", file.Cooldowns) + } + filtered := requestAuthFilesCooldowns(t, h, "?name=shared.json&auth_index="+url.QueryEscape(file.AuthIndex)) + if len(filtered.Files) != 1 || filtered.Files[0].ID != file.ID || string(filtered.Files[0].Cooldowns) != "[]" { + t.Fatalf("filter mismatch: %+v", filtered.Files) + } + } + if missing := requestAuthFilesCooldowns(t, h, "?auth_index=missing"); len(missing.Files) != 0 { + t.Fatal("unknown index matched") + } +} + +func TestListAuthFilesCooldownsUnknown(t *testing.T) { + t.Setenv("MANAGEMENT_PASSWORD", "") + for _, mode := range []string{"disk", "home"} { + t.Run(mode, func(t *testing.T) { + cfg := &config.Config{AuthDir: t.TempDir()} + var manager *coreauth.Manager + if mode == "disk" { + if errWrite := os.WriteFile(filepath.Join(cfg.AuthDir, "a.json"), []byte(`{"type":"codex"}`), 0o600); errWrite != nil { + t.Fatal(errWrite) + } + } else { + cfg.Home.Enabled = true + manager = coreauth.NewManager(nil, nil, nil) + manager.SetConfig(cfg) + registerAuthForLookupTest(t, manager, &coreauth.Auth{ID: "a", Provider: "codex", Status: coreauth.StatusActive, Attributes: map[string]string{"runtime_only": "true"}}) + } + h := NewHandlerWithoutConfigFilePath(cfg, manager) + payload := requestAuthFilesCooldowns(t, h, "") + if len(payload.Files) != 1 || string(payload.Files[0].Cooldowns) != "null" { + t.Fatalf("unknown cooldowns = %+v", payload.Files) + } + if mode == "disk" { + if filtered := requestAuthFilesCooldowns(t, h, "?auth_index=missing"); len(filtered.Files) != 0 { + t.Fatal("disk fallback matched index") + } + } + }) + } +} diff --git a/sdk/cliproxy/auth/cooldown_view.go b/sdk/cliproxy/auth/cooldown_view.go new file mode 100644 index 000000000..f42f2f125 --- /dev/null +++ b/sdk/cliproxy/auth/cooldown_view.go @@ -0,0 +1,171 @@ +package auth + +import ( + "sort" + "strings" + "time" +) + +// CooldownView describes an unexpired local retry restriction, not overall +// credential availability. It contains no credential metadata or raw errors. +type CooldownView struct { + Scope string `json:"scope"` + ModelKey string `json:"model_key,omitempty"` + Reason string `json:"reason"` + RetryAt time.Time `json:"retry_at"` + RemainingSeconds int64 `json:"remaining_seconds"` + BackoffLevel *int `json:"backoff_level,omitempty"` + HTTPStatus int `json:"http_status,omitempty"` +} + +// CooldownSnapshotForAuth projects a detached auth snapshot without mutating it. +// It reports timers even when another restriction (such as disablement or an +// expired token) also prevents execution. An empty result does not imply that +// the credential is usable. Callers must handle unavailable/remote state separately. +func CooldownSnapshotForAuth(auth *Auth, now time.Time) []CooldownView { + views := make([]CooldownView, 0) + if auth == nil { + return views + } + // Match the explicit credential-wide gate in isAuthBlockedForModel. Other + // auth-level fields can be model aggregates and must not become global gates. + if auth.Quota.Exceeded && auth.Quota.Reason == "credential_quota" && auth.Quota.NextRecoverAt.After(now) { + views = append(views, newCooldownView("credential", "", auth.Quota.NextRecoverAt, now, auth.Quota, auth.StatusMessage, auth.LastError)) + } else if len(auth.ModelStates) == 0 { + if blocked, _, next := availabilityBlock(auth.Unavailable, auth.Quota.Exceeded, auth.NextRetryAfter, auth.Quota.NextRecoverAt, now); blocked && next.After(now) { + views = append(views, newCooldownView("credential", "", next, now, auth.Quota, auth.StatusMessage, auth.LastError)) + } + } + + // Sorting source keys makes ties deterministic after selector precedence. + keys := make([]string, 0, len(auth.ModelStates)) + for key := range auth.ModelStates { + keys = append(keys, key) + } + sort.Strings(keys) + type modelCooldown struct { + view CooldownView + reason blockReason + } + byModel := make(map[string]modelCooldown) + for _, key := range keys { + state := auth.ModelStates[key] + model := canonicalModelKey(key) + if state == nil || model == "" { + continue + } + blocked, reason, next := availabilityBlock(state.Unavailable, state.Quota.Exceeded, state.NextRetryAfter, state.Quota.NextRecoverAt, now) + if !blocked || !next.After(now) { + continue + } + if previous, ok := byModel[model]; ok { + preferQuotaTie := next.Equal(previous.view.RetryAt) && reason == blockReasonCooldown && previous.reason != blockReasonCooldown + if !next.After(previous.view.RetryAt) && !preferQuotaTie { + continue + } + } + byModel[model] = modelCooldown{ + view: newCooldownView("model", model, next, now, state.Quota, state.StatusMessage, state.LastError), + reason: reason, + } + } + models := make([]string, 0, len(byModel)) + for model := range byModel { + models = append(models, model) + } + sort.Strings(models) + for _, model := range models { + views = append(views, byModel[model].view) + } + return views +} + +func newCooldownView(scope, model string, next, now time.Time, quota QuotaState, statusMessage string, lastErr *Error) CooldownView { + remaining := next.Sub(now) + seconds := int64(remaining / time.Second) + if remaining%time.Second != 0 { + seconds++ + } + view := CooldownView{ + Scope: scope, ModelKey: model, Reason: "unknown", + RetryAt: next.UTC(), RemainingSeconds: seconds, + } + // A shorter quota window must not label a longer non-quota retry timer. + // Zero recovery times can occur in legacy quota state with only a retry time. + if quota.Exceeded && (quota.NextRecoverAt.IsZero() || !quota.NextRecoverAt.Before(next)) { + switch quota.Reason { + case "credential_quota", "quota": + view.Reason = quota.Reason + case "cloudflare challenge": + view.Reason = "cloudflare_challenge" + } + } + propagatedQuota := quota.Exceeded && quota.Reason == "credential_quota" + if view.Reason == "credential_quota" { + // The credential-wide gate takes precedence over stale sibling errors. + return view + } + if (view.Reason == "quota" || view.Reason == "cloudflare_challenge") && quota.BackoffLevel >= 0 { + level := quota.BackoffLevel + view.BackoffLevel = &level + } + errorReason := cooldownErrorReason(lastErr) + if view.Reason == "unknown" { + view.Reason = errorReason + } + if view.Reason == "unknown" { + view.Reason = cooldownStatusReason(statusMessage) + } + // Propagation does not replace sibling errors. Avoid attributing a stale + // error status to that quota failure, even if a longer retry timer survives. + if !propagatedQuota && lastErr != nil && lastErr.HTTPStatus >= 400 && lastErr.HTTPStatus <= 599 && errorReason == view.Reason { + view.HTTPStatus = lastErr.HTTPStatus + } + return view +} + +func cooldownErrorReason(err *Error) string { + switch { + case isModelSupportResultError(err): + return "model_not_supported" + case isCloudflareChallengeResultError(err): + return "cloudflare_challenge" + case isInvalidGrantResultError(err): + return "invalid_grant" + } + switch statusCodeFromResult(err) { + case 401: + return "unauthorized" + case 402, 403: + return "payment_required" + case 404: + return "not_found" + case 429: + return "quota" + case 408, 500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526: + return "transient_error" + } + if err != nil { + return cooldownStatusReason(err.Code) + } + return "unknown" +} + +func cooldownStatusReason(message string) string { + // Only exact known markers can become public reason codes. Never return + // arbitrary status messages, error codes, or upstream response bodies. + switch strings.TrimSpace(message) { + case "quota", "quota exhausted": + return "quota" + case "cloudflare challenge": + return "cloudflare_challenge" + case "invalid_grant", "unauthorized", "payment_required", "not_found": + return strings.TrimSpace(message) + case "model_not_supported": + return "model_not_supported" + case "transient upstream error": + return "transient_error" + default: + return "unknown" + } +} diff --git a/sdk/cliproxy/auth/cooldown_view_test.go b/sdk/cliproxy/auth/cooldown_view_test.go new file mode 100644 index 000000000..4af80cc0d --- /dev/null +++ b/sdk/cliproxy/auth/cooldown_view_test.go @@ -0,0 +1,232 @@ +package auth + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + "time" +) + +func TestCooldownSnapshotForAuthScopes(t *testing.T) { + now := time.Date(2026, 7, 17, 10, 0, 0, 0, time.UTC) + modelState := func(after time.Duration) *ModelState { + return &ModelState{Unavailable: true, NextRetryAfter: now.Add(after), Quota: QuotaState{ + Exceeded: true, Reason: "quota", NextRecoverAt: now.Add(after), BackoffLevel: 6, + }, LastError: &Error{HTTPStatus: 429}} + } + for _, tt := range []struct { + name string + auth *Auth + want []string + }{ + {name: "nil", want: []string{}}, + {name: "empty", auth: &Auth{}, want: []string{}}, + {name: "aggregate is not credential cooldown", auth: &Auth{ + Unavailable: true, NextRetryAfter: now.Add(time.Minute), + Quota: QuotaState{Exceeded: true, Reason: "quota", NextRecoverAt: now.Add(time.Minute)}, + ModelStates: map[string]*ModelState{"b": modelState(time.Minute), "a": modelState(32 * time.Second), "ready": {}}, + }, want: []string{"model:a", "model:b"}}, + {name: "credential quota and longer model coexist", auth: &Auth{ + NextRetryAfter: now.Add(time.Hour), + Quota: QuotaState{Exceeded: true, Reason: "credential_quota", NextRecoverAt: now.Add(20 * time.Second), BackoffLevel: 9}, + ModelStates: map[string]*ModelState{"a": modelState(time.Minute)}, + }, want: []string{"credential:", "model:a"}}, + {name: "credential fallback", auth: &Auth{ + Unavailable: true, NextRetryAfter: now.Add(time.Minute), LastError: &Error{HTTPStatus: 503}, + }, want: []string{"credential:"}}, + {name: "nil model states still suppress fallback like selector", auth: &Auth{ + Unavailable: true, NextRetryAfter: now.Add(time.Minute), ModelStates: map[string]*ModelState{"a": nil}, + }, want: []string{}}, + {name: "disabled credential retains timer", auth: &Auth{ + Disabled: true, Status: StatusDisabled, Unavailable: true, NextRetryAfter: now.Add(time.Minute), + }, want: []string{"credential:"}}, + {name: "expired token does not erase timer", auth: &Auth{ + Metadata: map[string]any{"expired": now.Add(-time.Hour).Format(time.RFC3339)}, + ModelStates: map[string]*ModelState{"a": modelState(time.Minute)}, + }, want: []string{"model:a"}}, + {name: "disabled without timer", auth: &Auth{Disabled: true, ModelStates: map[string]*ModelState{"a": {Status: StatusDisabled}}}, want: []string{}}, + {name: "forced timer survives disable cooling override", auth: &Auth{ + Metadata: map[string]any{"disable_cooling": true}, Unavailable: true, NextRetryAfter: now.Add(time.Minute), + LastError: &Error{Code: ErrorCodeForceCooldown}, + }, want: []string{"credential:"}}, + } { + t.Run(tt.name, func(t *testing.T) { + got := CooldownSnapshotForAuth(tt.auth, now) + if got == nil { + t.Fatal("expected non-nil slice") + } + keys := make([]string, 0, len(got)) + for _, view := range got { + keys = append(keys, view.Scope+":"+view.ModelKey) + } + if !reflect.DeepEqual(keys, tt.want) { + t.Fatalf("scopes = %v, want %v", keys, tt.want) + } + if tt.name == "credential quota and longer model coexist" { + if got[0].RemainingSeconds != 20 || got[0].BackoffLevel != nil || got[0].HTTPStatus != 0 { + t.Fatalf("credential gate did not use its own deadline/diagnostics: %+v", got[0]) + } + } + }) + } +} + +func TestCooldownSnapshotForAuthTimeBoundaries(t *testing.T) { + now := time.Date(2026, 7, 17, 10, 0, 0, 0, time.UTC) + for _, tt := range []struct { + name string + state ModelState + seconds int64 + }{ + {name: "just before expiry", state: ModelState{Unavailable: true, NextRetryAfter: now.Add(time.Nanosecond)}, seconds: 1}, + {name: "fraction rounds up", state: ModelState{Unavailable: true, NextRetryAfter: now.Add(1500 * time.Millisecond)}, seconds: 2}, + {name: "exact expiry", state: ModelState{Unavailable: true, NextRetryAfter: now}}, + {name: "past expiry", state: ModelState{Unavailable: true, NextRetryAfter: now.Add(-time.Nanosecond)}}, + {name: "historical error and backoff", state: ModelState{Status: StatusError, Quota: QuotaState{BackoffLevel: 6}}}, + {name: "no deadline", state: ModelState{Unavailable: true, Quota: QuotaState{Exceeded: true}}}, + {name: "inactive future timestamp", state: ModelState{NextRetryAfter: now.Add(time.Minute)}}, + {name: "later quota time", state: ModelState{Unavailable: true, NextRetryAfter: now.Add(time.Second), Quota: QuotaState{Exceeded: true, NextRecoverAt: now.Add(3 * time.Second)}}, seconds: 3}, + {name: "later retry time", state: ModelState{Unavailable: true, NextRetryAfter: now.Add(4 * time.Second), Quota: QuotaState{Exceeded: true, NextRecoverAt: now.Add(time.Second)}}, seconds: 4}, + {name: "quota only", state: ModelState{Quota: QuotaState{Exceeded: true, NextRecoverAt: now.Add(5 * time.Second)}}, seconds: 5}, + {name: "expired quota and retry", state: ModelState{Unavailable: true, NextRetryAfter: now, Quota: QuotaState{Exceeded: true, NextRecoverAt: now, BackoffLevel: 6}}}, + {name: "retry hint independent of backoff", state: ModelState{Unavailable: true, NextRetryAfter: now.Add(97 * time.Second), Quota: QuotaState{Exceeded: true, Reason: "quota", BackoffLevel: 2}}, seconds: 97}, + } { + t.Run(tt.name, func(t *testing.T) { + got := CooldownSnapshotForAuth(&Auth{ModelStates: map[string]*ModelState{"a": &tt.state}}, now) + if tt.seconds == 0 { + if len(got) != 0 { + t.Fatalf("unexpected cooldown: %+v", got) + } + return + } + if len(got) != 1 || got[0].RemainingSeconds != tt.seconds { + t.Fatalf("views = %+v, want remaining %d", got, tt.seconds) + } + }) + } +} + +func TestCooldownSnapshotForAuthReasons(t *testing.T) { + now := time.Date(2026, 7, 17, 10, 0, 0, 0, time.UTC) + for _, tt := range []struct { + name string + quota QuotaState + err *Error + message string + reason string + status int + backoff bool + }{ + {name: "quota", quota: QuotaState{Exceeded: true, Reason: "quota", BackoffLevel: 6}, err: &Error{HTTPStatus: 429}, reason: "quota", status: 429, backoff: true}, + {name: "propagated quota hides old error", quota: QuotaState{Exceeded: true, Reason: "credential_quota", BackoffLevel: 6}, err: &Error{HTTPStatus: 401}, reason: "credential_quota"}, + {name: "quota hides unrelated error", quota: QuotaState{Exceeded: true, Reason: "quota"}, err: &Error{HTTPStatus: 503}, reason: "quota", backoff: true}, + {name: "challenge", quota: QuotaState{Exceeded: true, Reason: "cloudflare challenge"}, err: &Error{HTTPStatus: 403, Message: "cf-mitigated: challenge"}, reason: "cloudflare_challenge", status: 403, backoff: true}, + {name: "model unsupported", err: &Error{HTTPStatus: 400, Message: "model not supported"}, reason: "model_not_supported", status: 400}, + {name: "invalid grant", err: &Error{HTTPStatus: 400, Message: "invalid_grant"}, reason: "invalid_grant", status: 400}, + {name: "unauthorized", err: &Error{HTTPStatus: 401}, reason: "unauthorized", status: 401}, + {name: "payment", err: &Error{HTTPStatus: 402}, reason: "payment_required", status: 402}, + {name: "forbidden", err: &Error{HTTPStatus: 403}, reason: "payment_required", status: 403}, + {name: "not found", err: &Error{HTTPStatus: 404}, reason: "not_found", status: 404}, + {name: "gateway not challenge", err: &Error{HTTPStatus: 520, Message: "cloudflare challenge"}, reason: "transient_error", status: 520}, + {name: "shorter active quota does not label longer retry", quota: QuotaState{Exceeded: true, Reason: "quota", NextRecoverAt: now.Add(20 * time.Second), BackoffLevel: 6}, err: &Error{HTTPStatus: 503}, reason: "transient_error", status: 503}, + {name: "longer quota supplies deadline", quota: QuotaState{Exceeded: true, Reason: "quota", NextRecoverAt: now.Add(2 * time.Minute), BackoffLevel: 6}, err: &Error{HTTPStatus: 503}, reason: "quota", backoff: true}, + {name: "shorter propagated quota preserves longer retry reason", quota: QuotaState{Exceeded: true, Reason: "credential_quota", NextRecoverAt: now.Add(20 * time.Second), BackoffLevel: 6}, err: &Error{HTTPStatus: 401}, reason: "unauthorized"}, + {name: "expired quota does not label new failure", quota: QuotaState{Exceeded: true, Reason: "quota", NextRecoverAt: now, BackoffLevel: 6}, err: &Error{HTTPStatus: 503}, reason: "transient_error", status: 503}, + {name: "known marker", message: "transient upstream error", reason: "transient_error"}, + {name: "unknown is sanitized", quota: QuotaState{Exceeded: true, Reason: "secret-quota"}, err: &Error{Code: "secret-code", Message: "secret-body"}, message: "secret-message", reason: "unknown"}, + {name: "unknown HTTP error", err: &Error{HTTPStatus: 418}, reason: "unknown", status: 418}, + } { + t.Run(tt.name, func(t *testing.T) { + state := &ModelState{Unavailable: true, NextRetryAfter: now.Add(time.Minute), Quota: tt.quota, LastError: tt.err, StatusMessage: tt.message} + got := CooldownSnapshotForAuth(&Auth{ModelStates: map[string]*ModelState{"a": state}}, now) + if len(got) != 1 { + t.Fatalf("views = %+v", got) + } + view := got[0] + if view.Reason != tt.reason || view.HTTPStatus != tt.status || (view.BackoffLevel != nil) != tt.backoff { + t.Fatalf("view = %+v, want reason=%s status=%d backoff=%v", view, tt.reason, tt.status, tt.backoff) + } + encoded, errMarshal := json.Marshal(view) + if errMarshal != nil { + t.Fatal(errMarshal) + } + if strings.Contains(string(encoded), "secret") { + t.Fatalf("raw data leaked: %s", encoded) + } + }) + } +} + +func TestCooldownSnapshotForAuthDeduplicatesWithoutMutation(t *testing.T) { + now := time.Date(2026, 7, 17, 10, 0, 0, 0, time.FixedZone("offset", 3600)) + auth := &Auth{ + Metadata: map[string]any{"access_token": "secret-token"}, + ModelStates: map[string]*ModelState{ + " model-a(high) ": {Unavailable: true, NextRetryAfter: now.Add(time.Minute), LastError: &Error{HTTPStatus: 503}}, + "model-a": {Unavailable: true, NextRetryAfter: now.Add(32 * time.Second), Quota: QuotaState{Exceeded: true, Reason: "quota", BackoffLevel: 6}, LastError: &Error{HTTPStatus: 429}}, + "model-b": {Unavailable: true, NextRetryAfter: now.Add(time.Minute), Quota: QuotaState{Exceeded: true, Reason: "quota", BackoffLevel: 6}}, + " ": {Unavailable: true, NextRetryAfter: now.Add(time.Minute)}, + "nil": nil, + }, + } + before := auth.Clone() + got := CooldownSnapshotForAuth(auth, now) + if len(got) != 2 || got[0].ModelKey != "model-a" || got[0].Reason != "transient_error" || got[0].HTTPStatus != 503 || got[0].BackoffLevel != nil || got[0].RemainingSeconds != 60 { + t.Fatalf("deduplicated views = %+v", got) + } + if got[0].RetryAt.Location() != time.UTC { + t.Fatal("retry_at is not UTC") + } + for range 20 { + if again := CooldownSnapshotForAuth(auth, now); !reflect.DeepEqual(again, got) { + t.Fatal("unstable snapshot") + } + } + *got[1].BackoffLevel = 999 + if !reflect.DeepEqual(auth, before) { + t.Fatal("projection or returned view mutated auth") + } +} + +func TestCooldownSnapshotForAuthEqualDeadlineIsStable(t *testing.T) { + now := time.Date(2026, 7, 17, 10, 0, 0, 0, time.UTC) + for _, quotaKey := range []string{"a(high)", "a(low)"} { + t.Run(quotaKey, func(t *testing.T) { + otherKey := "a(low)" + if quotaKey == otherKey { + otherKey = "a(high)" + } + auth := &Auth{ModelStates: map[string]*ModelState{ + quotaKey: {Unavailable: true, NextRetryAfter: now.Add(time.Minute), LastError: &Error{HTTPStatus: 429}, Quota: QuotaState{Exceeded: true, Reason: "quota", NextRecoverAt: now.Add(time.Minute)}}, + otherKey: {Unavailable: true, NextRetryAfter: now.Add(time.Minute), LastError: &Error{HTTPStatus: 503}}, + }} + blocked, reason, next := isAuthBlockedForModel(auth, "a", now) + if !blocked || reason != blockReasonCooldown { + t.Fatal("precondition: selector must prefer quota on equal deadlines") + } + for range 20 { + got := CooldownSnapshotForAuth(auth, now) + if len(got) != 1 || got[0].Reason != "quota" || !got[0].RetryAt.Equal(next) || got[0].HTTPStatus != 429 { + t.Fatalf("unstable equal-deadline selection: %+v", got) + } + } + }) + } +} + +func TestCooldownSnapshotForAuthLongerRetryReasonSurvivesQuotaExpiry(t *testing.T) { + now := time.Date(2026, 7, 17, 10, 0, 0, 0, time.UTC) + auth := &Auth{ModelStates: map[string]*ModelState{ + "a": { + Unavailable: true, NextRetryAfter: now.Add(time.Hour), LastError: &Error{HTTPStatus: 503}, + Quota: QuotaState{Exceeded: true, Reason: "quota", NextRecoverAt: now.Add(5 * time.Minute), BackoffLevel: 6}, + }, + }} + for _, observed := range []time.Time{now, now.Add(5 * time.Minute), now.Add(6 * time.Minute)} { + got := CooldownSnapshotForAuth(auth, observed) + if len(got) != 1 || got[0].Reason != "transient_error" || got[0].HTTPStatus != 503 || got[0].BackoffLevel != nil || !got[0].RetryAt.Equal(now.Add(time.Hour)) { + t.Fatalf("longer retry diagnostics changed at %v: %+v", observed, got) + } + } +}