From 4da9f68527611a2d88b3cc6a9d30e38e1d0b846a Mon Sep 17 00:00:00 2001 From: Luis Pater Date: Sat, 14 Nov 2026 03:53:41 +0800 Subject: [PATCH] fix(auth): carry the pool cooldown reset on the terminal 429 - Wrap the final 429 in a poolCooldownTerminalError at the Execute/ExecuteCount/ExecuteStream boundaries when every candidate credential for the model is blocked and at least one is in a timed cooldown, preserving the upstream error body and status while marking IsModelCooldown so the Claude handler appends the reset time. - Compute the Retry-After from the earliest auto-recovery across all blocked candidates (including non-cooldown timed blocks) so the client sees the pool's real recovery instead of the last attempt's hint. - Add regression coverage for both cooldown variants across the three execution paths, the available-candidate guard, mixed-block earliest deadlines, the sub-floor hint clamp, and the Claude handler end-to-end. Closes: #7521 --- .../code_handlers_pool_cooldown_test.go | 174 +++++++ sdk/cliproxy/auth/conductor_cooldown.go | 3 +- sdk/cliproxy/auth/conductor_execution.go | 13 + .../auth/conductor_pool_cooldown_test.go | 459 ++++++++++++++++++ sdk/cliproxy/auth/conductor_selection.go | 138 ++++++ sdk/cliproxy/auth/errors.go | 91 ++++ sdk/cliproxy/auth/home_concurrency.go | 4 + 7 files changed, 881 insertions(+), 1 deletion(-) create mode 100644 sdk/api/handlers/claude/code_handlers_pool_cooldown_test.go create mode 100644 sdk/cliproxy/auth/conductor_pool_cooldown_test.go diff --git a/sdk/api/handlers/claude/code_handlers_pool_cooldown_test.go b/sdk/api/handlers/claude/code_handlers_pool_cooldown_test.go new file mode 100644 index 0000000000..6ee677bcc7 --- /dev/null +++ b/sdk/api/handlers/claude/code_handlers_pool_cooldown_test.go @@ -0,0 +1,174 @@ +package claude + +import ( + "context" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "github.com/MoeForge/moego/ojson" + "github.com/gin-gonic/gin" + "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" + "github.com/router-for-me/CLIProxyAPI/v8/sdk/api/handlers" + coreauth "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/auth" + coreexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" + sdkconfig "github.com/router-for-me/CLIProxyAPI/v8/sdk/config" +) + +type issue7521UpstreamError struct { + status int + body string + retryAfter time.Duration + headers http.Header +} + +func (e *issue7521UpstreamError) Error() string { return e.body } +func (e *issue7521UpstreamError) StatusCode() int { return e.status } +func (e *issue7521UpstreamError) RetryAfter() *time.Duration { + d := e.retryAfter + return &d +} +func (e *issue7521UpstreamError) Headers() http.Header { + if e.headers == nil { + return nil + } + return e.headers.Clone() +} + +type issue7521ClaudeExecutor struct { + executeFn func(ctx context.Context, auth *coreauth.Auth, req coreexecutor.Request, opts coreexecutor.Options) (coreexecutor.Response, error) + streamFn func(ctx context.Context, auth *coreauth.Auth, req coreexecutor.Request, opts coreexecutor.Options) (*coreexecutor.StreamResult, error) + countFn func(ctx context.Context, auth *coreauth.Auth, req coreexecutor.Request, opts coreexecutor.Options) (coreexecutor.Response, error) +} + +func (e *issue7521ClaudeExecutor) Identifier() string { return "claude" } + +func (e *issue7521ClaudeExecutor) Execute(ctx context.Context, auth *coreauth.Auth, req coreexecutor.Request, opts coreexecutor.Options) (coreexecutor.Response, error) { + coreexecutor.MarkUpstreamAttempt(ctx) + return e.executeFn(ctx, auth, req, opts) +} + +func (e *issue7521ClaudeExecutor) ExecuteStream(ctx context.Context, auth *coreauth.Auth, req coreexecutor.Request, opts coreexecutor.Options) (*coreexecutor.StreamResult, error) { + coreexecutor.MarkUpstreamAttempt(ctx) + return e.streamFn(ctx, auth, req, opts) +} + +func (e *issue7521ClaudeExecutor) CountTokens(ctx context.Context, auth *coreauth.Auth, req coreexecutor.Request, opts coreexecutor.Options) (coreexecutor.Response, error) { + coreexecutor.MarkUpstreamAttempt(ctx) + return e.countFn(ctx, auth, req, opts) +} + +func (e *issue7521ClaudeExecutor) Refresh(_ context.Context, auth *coreauth.Auth) (*coreauth.Auth, error) { + return auth, nil +} + +func (e *issue7521ClaudeExecutor) HttpRequest(_ context.Context, _ *coreauth.Auth, _ *http.Request) (*http.Response, error) { + return nil, nil +} + +func TestClaudeLastCredential429AppendsPoolResetTime(t *testing.T) { + gin.SetMode(gin.TestMode) + + const model = "issue-7521-claude-route-model" + shortHint := 95 * time.Minute + longHint := 3 * time.Hour + + makeErr := func(authID string) error { + if strings.HasSuffix(authID, "-a") { + return &issue7521UpstreamError{ + status: http.StatusTooManyRequests, + body: `{"type":"error","error":{"type":"rate_limit_error","message":"First account rate limited."}}`, + retryAfter: shortHint, + headers: http.Header{"Retry-After": {"5700"}}, + } + } + return &issue7521UpstreamError{ + status: http.StatusTooManyRequests, + body: `{"type":"error","error":{"type":"rate_limit_error","message":"This request would exceed your account's monthly spend limit. Please try again later."}}`, + retryAfter: longHint, + headers: http.Header{"Retry-After": {"10800"}, "X-Upstream-Trace": {"trace-b"}}, + } + } + + for _, tc := range []struct { + name string + path string + streaming bool + }{ + {name: "messages_non_stream", path: "/v1/messages", streaming: false}, + {name: "messages_stream", path: "/v1/messages", streaming: true}, + {name: "count_tokens", path: "/v1/messages/count_tokens", streaming: false}, + } { + t.Run(tc.name, func(t *testing.T) { + manager := coreauth.NewManager(nil, nil, nil) + manager.RegisterExecutor(&issue7521ClaudeExecutor{ + executeFn: func(_ context.Context, auth *coreauth.Auth, _ coreexecutor.Request, _ coreexecutor.Options) (coreexecutor.Response, error) { + return coreexecutor.Response{}, makeErr(auth.ID) + }, + streamFn: func(_ context.Context, auth *coreauth.Auth, _ coreexecutor.Request, _ coreexecutor.Options) (*coreexecutor.StreamResult, error) { + chunks := make(chan coreexecutor.StreamChunk, 1) + chunks <- coreexecutor.StreamChunk{Err: makeErr(auth.ID)} + close(chunks) + return &coreexecutor.StreamResult{Chunks: chunks}, nil + }, + countFn: func(_ context.Context, auth *coreauth.Auth, _ coreexecutor.Request, _ coreexecutor.Options) (coreexecutor.Response, error) { + return coreexecutor.Response{}, makeErr(auth.ID) + }, + }) + + reg := registry.GetGlobalRegistry() + for _, suffix := range []string{"-a", "-b"} { + authID := "issue-7521-route-" + tc.name + suffix + auth := &coreauth.Auth{ + ID: authID, + Provider: "claude", + Status: coreauth.StatusActive, + Attributes: map[string]string{"api_key": "key" + suffix}, + } + if _, errReg := manager.Register(context.Background(), auth); errReg != nil { + t.Fatalf("register auth: %v", errReg) + } + reg.RegisterClient(authID, "claude", []*registry.ModelInfo{{ID: model}}) + t.Cleanup(func() { + reg.UnregisterClient(authID) + }) + } + + handler := NewClaudeCodeAPIHandler(handlers.NewBaseAPIHandlers(&sdkconfig.SDKConfig{PassthroughHeaders: true}, manager)) + router := gin.New() + router.POST("/v1/messages", handler.ClaudeMessages) + router.POST("/v1/messages/count_tokens", handler.ClaudeCountTokens) + + body := `{"model":"` + model + `","messages":[{"role":"user","content":"hi"}],"stream":` + strconv.FormatBool(tc.streaming) + `}` + req := httptest.NewRequest(http.MethodPost, tc.path, strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusTooManyRequests { + t.Fatalf("status = %d, want 429; body=%s", rec.Code, rec.Body.String()) + } + seconds, errParse := strconv.ParseInt(rec.Header().Get("Retry-After"), 10, 64) + if errParse != nil || seconds <= 0 { + t.Fatalf("Retry-After = %q, want positive integer seconds", rec.Header().Get("Retry-After")) + } + if seconds > int64((shortHint + 10*time.Second).Seconds()) { + t.Fatalf("Retry-After = %d, want pool deadline near %d rather than last attempt 10800", seconds, int64(shortHint.Seconds())) + } + if gotTrace := rec.Header().Get("X-Upstream-Trace"); gotTrace != "trace-b" { + t.Fatalf("X-Upstream-Trace = %q, want trace-b preserved under passthrough-headers", gotTrace) + } + if gotType := ojson.GetBytesNoCopy(rec.Body.Bytes(), "error.type").String(); gotType != "rate_limit_error" { + t.Fatalf("error.type = %q, want rate_limit_error", gotType) + } + wantDelay := (time.Duration(seconds) * time.Second).String() + wantMessage := "This request would exceed your account's monthly spend limit. Please try again later. (reset after " + wantDelay + ")" + if gotMsg := ojson.GetBytesNoCopy(rec.Body.Bytes(), "error.message").String(); gotMsg != wantMessage { + t.Fatalf("error.message = %q, want %q", gotMsg, wantMessage) + } + }) + } +} diff --git a/sdk/cliproxy/auth/conductor_cooldown.go b/sdk/cliproxy/auth/conductor_cooldown.go index e7881fc529..21705f19d3 100644 --- a/sdk/cliproxy/auth/conductor_cooldown.go +++ b/sdk/cliproxy/auth/conductor_cooldown.go @@ -3455,7 +3455,8 @@ func retryAfterFromError(err error) *time.Duration { // An expired trusted wrapper reports a negative remaining time; natural // expiry is not an invalid provider hint, so fall through to the wrapped // error's own hint instead of surfacing the negative value. - if _, isTrusted := rap.(*trustedRetryAfterError); isTrusted { + switch rap.(type) { + case *poolCooldownTerminalError, *trustedRetryAfterError: if inner := errors.Unwrap(err); inner != nil { return retryAfterFromError(inner) } diff --git a/sdk/cliproxy/auth/conductor_execution.go b/sdk/cliproxy/auth/conductor_execution.go index a05f57e584..b092c81d67 100644 --- a/sdk/cliproxy/auth/conductor_execution.go +++ b/sdk/cliproxy/auth/conductor_execution.go @@ -224,6 +224,7 @@ func (m *Manager) Execute(ctx context.Context, providers []string, req cliproxye var lastErr error var preferredUpstreamErr error + var selectionCooldownDeadline time.Time var terminalRoundRejectedByScheduler bool retryModel := authSelectionModelFromOptions(opts, req.Model) for attempt := 0; ; attempt++ { @@ -244,6 +245,7 @@ func (m *Manager) Execute(ctx context.Context, providers []string, req cliproxye break } terminalRoundRejectedByScheduler = hasRoundEndedBySchedulerReject(errExec) + observeSelectionCooldownDeadline(&selectionCooldownDeadline, errExec, time.Now()) if hasUpstreamExecutionAttempt(errExec) { preferredUpstreamErr = errExec } @@ -272,6 +274,7 @@ func (m *Manager) Execute(ctx context.Context, providers []string, req cliproxye return resp, nil } } + lastErr = m.annotatePoolCooldownTerminalError(ctx, normalized, retryModel, opts, lastErr, selectionCooldownDeadline) return cliproxyexecutor.Response{}, lastErr } return cliproxyexecutor.Response{}, &Error{Code: "auth_not_found", Message: "no auth available"} @@ -299,6 +302,7 @@ func (m *Manager) ExecuteCount(ctx context.Context, providers []string, req clip var lastErr error var preferredUpstreamErr error + var selectionCooldownDeadline time.Time retryModel := authSelectionModelFromOptions(opts, req.Model) for attempt := 0; ; attempt++ { roundAttempted := make(map[string]struct{}) @@ -316,6 +320,7 @@ func (m *Manager) ExecuteCount(ctx context.Context, providers []string, req clip } break } + observeSelectionCooldownDeadline(&selectionCooldownDeadline, errExec, time.Now()) if hasUpstreamExecutionAttempt(errExec) { preferredUpstreamErr = errExec } @@ -337,6 +342,7 @@ func (m *Manager) ExecuteCount(ctx context.Context, providers []string, req clip lastErr = preferredExecutionAttemptError(lastErr, preferredUpstreamErr) lastErr = unwrapExecutionBoundaryError(lastErr) lastErr = m.normalizeSessionAffinityNeverTerminalError(lastErr, normalized, retryModel, opts) + lastErr = m.annotatePoolCooldownTerminalError(ctx, normalized, retryModel, opts, lastErr, selectionCooldownDeadline) return cliproxyexecutor.Response{}, lastErr } return cliproxyexecutor.Response{}, &Error{Code: "auth_not_found", Message: "no auth available"} @@ -375,6 +381,7 @@ func (m *Manager) ExecuteStream(ctx context.Context, providers []string, req cli var lastErr error var preferredUpstreamErr error + var selectionCooldownDeadline time.Time var terminalRoundRejectedByScheduler bool homeRetryLimit := -1 retryModel := authSelectionModelFromOptions(opts, req.Model) @@ -388,6 +395,9 @@ func (m *Manager) ExecuteStream(ctx context.Context, providers []string, req cli if errStream == nil { return result, nil } + if !m.HomeEnabled() { + observeSelectionCooldownDeadline(&selectionCooldownDeadline, errStream, time.Now()) + } if hasUpstreamExecutionAttempt(errStream) { preferredUpstreamErr = errStream } @@ -446,6 +456,9 @@ func (m *Manager) ExecuteStream(ctx context.Context, providers []string, req cli return result, nil } } + if !m.HomeEnabled() { + lastErr = m.annotatePoolCooldownTerminalError(ctx, normalized, retryModel, opts, lastErr, selectionCooldownDeadline) + } var bootstrapErr *streamBootstrapError if errors.As(lastErr, &bootstrapErr) && bootstrapErr != nil { return streamErrorResult(bootstrapErr.Headers(), lastErr), nil diff --git a/sdk/cliproxy/auth/conductor_pool_cooldown_test.go b/sdk/cliproxy/auth/conductor_pool_cooldown_test.go new file mode 100644 index 0000000000..87302417c2 --- /dev/null +++ b/sdk/cliproxy/auth/conductor_pool_cooldown_test.go @@ -0,0 +1,459 @@ +package auth + +import ( + "context" + "errors" + "net/http" + "strconv" + "strings" + "testing" + "time" + + "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" + cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" +) + +const issue7521PoolModel = "claude-sonnet-4-6-issue-7521" + +func newIssue7521Manager( + t *testing.T, + selector Selector, + exec ProviderExecutor, + auths ...*Auth, +) *Manager { + t.Helper() + withQuotaCooldownEnabled(t) + if selector == nil { + selector = &RoundRobinSelector{} + } + manager := NewManager(nil, selector, nil) + manager.SetRetryConfig(2, 30*time.Second, 0) + manager.RegisterExecutor(exec) + reg := registry.GetGlobalRegistry() + for _, auth := range auths { + authID := auth.ID + reg.RegisterClient(authID, auth.Provider, []*registry.ModelInfo{{ID: issue7521PoolModel}}) + t.Cleanup(func() { + reg.UnregisterClient(authID) + }) + if _, errRegister := manager.Register(context.Background(), auth); errRegister != nil { + t.Fatalf("register auth %s: %v", authID, errRegister) + } + } + return manager +} + +func newIssue7521Auth(id string) *Auth { + return &Auth{ + ID: id, + Provider: "claude", + Status: StatusActive, + Attributes: map[string]string{"api_key": "key-" + id}, + } +} + +func newIssue7521Upstream429(msg string, retryAfter time.Duration) error { + hint := retryAfter + return customStatusError{ + code: http.StatusTooManyRequests, + msg: msg, + retryAfter: &hint, + } +} + +func assertIssue7521PoolCooldownError(t *testing.T, err error, wantMessage string, wantDeadline time.Duration) { + t.Helper() + if err == nil { + t.Fatal("expected terminal 429 error, got nil") + } + if got := statusCodeFromError(err); got != http.StatusTooManyRequests { + t.Fatalf("status = %d, want 429", got) + } + if !strings.Contains(err.Error(), wantMessage) { + t.Fatalf("error = %q, want upstream message %q preserved", err.Error(), wantMessage) + } + var marker interface{ IsModelCooldown() bool } + if !errors.As(err, &marker) || marker == nil || !marker.IsModelCooldown() { + t.Fatalf("error %T (%v) must report IsModelCooldown() == true", err, err) + } + if !HasTrustedRetryDeadline(err) { + t.Fatal("HasTrustedRetryDeadline(err) = false, want true") + } + retryAfter := SafeResponseHeaders(err).Get("Retry-After") + seconds, errParse := strconv.ParseInt(retryAfter, 10, 64) + if errParse != nil || seconds <= 0 { + t.Fatalf("Retry-After = %q, want positive integer seconds", retryAfter) + } + gotDuration := time.Duration(seconds) * time.Second + if gotDuration > wantDeadline+10*time.Second || gotDuration < wantDeadline-10*time.Second { + t.Fatalf("Retry-After = %ds (%s), want near pool recovery %s", seconds, gotDuration, wantDeadline) + } +} + +// Variant 2: fresh start where every candidate credential is tried in the same +// round and returns 429; no modelCooldownError is built during selection, yet +// the terminal 429 must report IsModelCooldown() and carry the earliest pool reset. +func TestExecuteLastCredential429CarriesEarliestPoolReset(t *testing.T) { + first := newIssue7521Auth("7521-exec-a") + second := newIssue7521Auth("7521-exec-b") + shortHint := 90 * time.Minute + longHint := 3 * time.Hour + + var attempts []string + exec := &customStreamMockExecutor{ + identifier: "claude", + mockCustomErrorExecutor: mockCustomErrorExecutor{ + identifier: "claude", + executeFn: func(ctx context.Context, auth *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { + cliproxyexecutor.MarkUpstreamAttempt(ctx) + attempts = append(attempts, auth.ID) + if auth.ID == first.ID { + return cliproxyexecutor.Response{}, newIssue7521Upstream429("first credential rate limited", shortHint) + } + return cliproxyexecutor.Response{}, newIssue7521Upstream429("monthly spend limit reached", longHint) + }, + }, + } + manager := newIssue7521Manager(t, nil, exec, first, second) + + _, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{}) + assertIssue7521PoolCooldownError(t, errExec, "monthly spend limit reached", shortHint) + if len(attempts) != 2 { + t.Fatalf("attempts = %v, want 2 credentials tried", attempts) + } +} + +func TestExecuteSingleCredentialSubFloorHintReportsActualPoolRecovery(t *testing.T) { + single := newIssue7521Auth("7521-subfloor-a") + hint := 7 * time.Second + exec := &customStreamMockExecutor{ + identifier: "claude", + mockCustomErrorExecutor: mockCustomErrorExecutor{ + identifier: "claude", + executeFn: func(ctx context.Context, _ *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { + cliproxyexecutor.MarkUpstreamAttempt(ctx) + return cliproxyexecutor.Response{}, newIssue7521Upstream429("try later", hint) + }, + }, + } + manager := newIssue7521Manager(t, nil, exec, single) + manager.SetRetryConfig(0, 0, 0) + + _, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{}) + if errExec == nil { + t.Fatal("expected terminal 429 error") + } + // MarkResult clamps the 7s hint up to minQuotaCooldownFloor (10s), so the + // pool's actual recovery is 10s out. The wrap must advertise that real + // deadline, not the shorter per-attempt hint. + if got := SafeResponseHeaders(errExec).Get("Retry-After"); got != "10" { + t.Fatalf("Retry-After = %q, want 10 (minQuotaCooldownFloor)", got) + } + var marker interface{ IsModelCooldown() bool } + if !errors.As(errExec, &marker) || marker == nil || !marker.IsModelCooldown() { + t.Fatalf("error %T must report IsModelCooldown() == true", errExec) + } +} + +// Variant 1: one credential is already cooling down; the remaining credential +// returns 429 and the next pick fails with modelCooldownError, which is +// replaced by preferredExecutionAttemptError. The terminal 429 must still keep +// the upstream text and carry the earliest pool cooldown reset. +func TestExecuteLastUsableCredential429KeepsExistingCooldownReset(t *testing.T) { + first := newIssue7521Auth("7521-var1-a") + second := newIssue7521Auth("7521-var1-b") + existingCooldown := 75 * time.Minute + freshHint := 3 * time.Hour + + exec := &customStreamMockExecutor{ + identifier: "claude", + mockCustomErrorExecutor: mockCustomErrorExecutor{ + identifier: "claude", + executeFn: func(ctx context.Context, _ *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { + cliproxyexecutor.MarkUpstreamAttempt(ctx) + return cliproxyexecutor.Response{}, newIssue7521Upstream429("second credential rate limited", freshHint) + }, + }, + } + manager := newIssue7521Manager(t, nil, exec, first, second) + manager.MarkResult(context.Background(), Result{ + AuthID: first.ID, + Provider: "claude", + Model: issue7521PoolModel, + Success: false, + RetryAfter: &existingCooldown, + CredentialScope: true, + Error: &Error{HTTPStatus: http.StatusTooManyRequests, Message: "already cooling"}, + }) + + _, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{}) + assertIssue7521PoolCooldownError(t, errExec, "second credential rate limited", existingCooldown) +} + +func TestExecuteCountLastCredential429CarriesEarliestPoolReset(t *testing.T) { + first := newIssue7521Auth("7521-count-a") + second := newIssue7521Auth("7521-count-b") + shortHint := 85 * time.Minute + longHint := 4 * time.Hour + + exec := &customStreamMockExecutor{ + identifier: "claude", + mockCustomErrorExecutor: mockCustomErrorExecutor{ + identifier: "claude", + countFn: func(ctx context.Context, auth *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { + cliproxyexecutor.MarkUpstreamAttempt(ctx) + if auth.ID == first.ID { + return cliproxyexecutor.Response{}, newIssue7521Upstream429("count first limited", shortHint) + } + return cliproxyexecutor.Response{}, newIssue7521Upstream429("count second limited", longHint) + }, + }, + } + manager := newIssue7521Manager(t, nil, exec, first, second) + + _, errCount := manager.ExecuteCount(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{}) + assertIssue7521PoolCooldownError(t, errCount, "count second limited", shortHint) +} + +func TestExecuteStreamLastCredential429CarriesEarliestPoolReset(t *testing.T) { + shortHint := 80 * time.Minute + longHint := 3 * time.Hour + + t.Run("direct stream error", func(t *testing.T) { + first := newIssue7521Auth("7521-stream-direct-a") + second := newIssue7521Auth("7521-stream-direct-b") + exec := &customStreamMockExecutor{ + identifier: "claude", + streamFn: func(ctx context.Context, auth *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) { + cliproxyexecutor.MarkUpstreamAttempt(ctx) + if auth.ID == first.ID { + return nil, newIssue7521Upstream429("stream first limited", shortHint) + } + return nil, newIssue7521Upstream429("stream second limited", longHint) + }, + } + manager := newIssue7521Manager(t, nil, exec, first, second) + + _, errStream := manager.ExecuteStream(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{Stream: true}) + assertIssue7521PoolCooldownError(t, errStream, "stream second limited", shortHint) + }) + + t.Run("bootstrap chunk error", func(t *testing.T) { + first := newIssue7521Auth("7521-stream-boot-a") + second := newIssue7521Auth("7521-stream-boot-b") + exec := &customStreamMockExecutor{ + identifier: "claude", + streamFn: func(ctx context.Context, auth *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) { + cliproxyexecutor.MarkUpstreamAttempt(ctx) + chunks := make(chan cliproxyexecutor.StreamChunk, 1) + if auth.ID == first.ID { + chunks <- cliproxyexecutor.StreamChunk{Err: newIssue7521Upstream429("bootstrap first limited", shortHint)} + } else { + chunks <- cliproxyexecutor.StreamChunk{Err: newIssue7521Upstream429("bootstrap second limited", longHint)} + } + close(chunks) + return &cliproxyexecutor.StreamResult{Chunks: chunks}, nil + }, + } + manager := newIssue7521Manager(t, nil, exec, first, second) + + streamRes, errStream := manager.ExecuteStream(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{Stream: true}) + if errStream != nil { + t.Fatalf("expected streamErrorResult, got direct err: %v", errStream) + } + chunk, ok := <-streamRes.Chunks + if !ok { + t.Fatal("expected error chunk on bootstrap failure") + } + assertIssue7521PoolCooldownError(t, chunk.Err, "bootstrap second limited", shortHint) + }) +} + +func TestExecute429LeavesOrdinary429UnchangedWhenCandidateAvailable(t *testing.T) { + first := newIssue7521Auth("7521-avail-a") + second := newIssue7521Auth("7521-avail-b") + hint := time.Hour + + exec := &customStreamMockExecutor{ + identifier: "claude", + mockCustomErrorExecutor: mockCustomErrorExecutor{ + identifier: "claude", + executeFn: func(ctx context.Context, _ *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { + cliproxyexecutor.MarkUpstreamAttempt(ctx) + return cliproxyexecutor.Response{}, newIssue7521Upstream429("single attempt 429", hint) + }, + }, + } + manager := newIssue7521Manager(t, nil, exec, first, second) + // Stop after 1 credential attempt with 0 retries so the second credential remains unblocked. + manager.SetRetryConfig(0, 0, 1) + + _, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{}) + if errExec == nil { + t.Fatal("expected 429 error") + } + if got := statusCodeFromError(errExec); got != http.StatusTooManyRequests { + t.Fatalf("status = %d, want 429", got) + } + var marker interface{ IsModelCooldown() bool } + if errors.As(errExec, &marker) && marker != nil && marker.IsModelCooldown() { + t.Fatalf("ordinary 429 must not report IsModelCooldown() when another candidate is still available: %T (%v)", errExec, errExec) + } +} + +func TestExecuteLastCredential429IgnoresZeroWeightCandidateUnderWeightedSelector(t *testing.T) { + active := newIssue7521Auth("7521-weight-active") + drained := newIssue7521Auth("7521-weight-zero") + drained.Attributes["weight"] = "0" + hint := 70 * time.Minute + + exec := &customStreamMockExecutor{ + identifier: "claude", + mockCustomErrorExecutor: mockCustomErrorExecutor{ + identifier: "claude", + executeFn: func(ctx context.Context, _ *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { + cliproxyexecutor.MarkUpstreamAttempt(ctx) + return cliproxyexecutor.Response{}, newIssue7521Upstream429("active credential exhausted", hint) + }, + }, + } + selector := NewSessionAffinitySelector(&WeightedRoundRobinSelector{}) + manager := newIssue7521Manager(t, selector, exec, active, drained) + + _, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{}) + assertIssue7521PoolCooldownError(t, errExec, "active credential exhausted", hint) +} + +func TestAnnotatePoolCooldownSelectionFallbackAndUnblockedGuard(t *testing.T) { + now := time.Date(2026, 11, 14, 12, 0, 0, 0, time.UTC) + fallbackDeadline := now.Add(42 * time.Minute) + lastAttemptHint := 3 * time.Hour + baseErr := withTrustedRetryAfter(newIssue7521Upstream429("upstream rate limited", lastAttemptHint), &lastAttemptHint) + + t.Run("falls back to selection cooldown when no candidate unblocked", func(t *testing.T) { + manager := NewManager(nil, nil, nil) + wrapped := manager.annotatePoolCooldownTerminalErrorAt( + context.Background(), + []string{"claude"}, + issue7521PoolModel, + cliproxyexecutor.Options{}, + baseErr, + fallbackDeadline, + now, + ) + poolErr, ok := wrapped.(*poolCooldownTerminalError) + if !ok || poolErr == nil { + t.Fatalf("wrapped = %T, want *poolCooldownTerminalError", wrapped) + } + if !poolErr.deadline.Equal(fallbackDeadline) { + t.Fatalf("deadline = %v, want %v", poolErr.deadline, fallbackDeadline) + } + if !poolErr.IsModelCooldown() { + t.Fatal("IsModelCooldown() = false, want true") + } + }) + + t.Run("does not fall back when an eligible candidate is unblocked", func(t *testing.T) { + avail := newIssue7521Auth("7521-fallback-avail") + exec := &customStreamMockExecutor{identifier: "claude"} + manager := newIssue7521Manager(t, nil, exec, avail) + + got := manager.annotatePoolCooldownTerminalErrorAt( + context.Background(), + []string{"claude"}, + issue7521PoolModel, + cliproxyexecutor.Options{}, + baseErr, + fallbackDeadline, + now, + ) + var marker interface{ IsModelCooldown() bool } + if errors.As(got, &marker) && marker != nil && marker.IsModelCooldown() { + t.Fatalf("must not wrap as model cooldown when a candidate is unblocked: %T", got) + } + }) + + t.Run("expired pool deadline suppresses raw header and falls back for retryAfterFromError", func(t *testing.T) { + cause := headerCarryingError{headers: http.Header{"X-Trace": {"1"}, "Retry-After": {"999"}}} + expired := &poolCooldownTerminalError{ + cause: cause, + deadline: time.Now().Add(-time.Second), + } + if !HasTrustedRetryDeadline(expired) { + t.Fatal("HasTrustedRetryDeadline(expired) = false, want true") + } + if got := SafeResponseHeaders(expired).Get("Retry-After"); got != "" { + t.Fatalf("SafeResponseHeaders(expired) Retry-After = %q, want empty", got) + } + if got := expired.Headers().Get("X-Trace"); got != "1" { + t.Fatalf("Headers() X-Trace = %q, want 1", got) + } + agg := 2 * time.Second + homeTerminal := markHomeRetryRoundExhausted(&poolCooldownTerminalError{ + cause: cause, + deadline: time.Now().Add(time.Hour), + }, &agg, false) + if got := SafeResponseHeaders(homeTerminal).Get("Retry-After"); got != "2" { + t.Fatalf("Home aggregate Retry-After = %q, want 2", got) + } + }) +} + +// Regression for the review finding: a short transient non-cooldown block on a +// sibling credential must pull the pool reset forward instead of being ignored, +// so the wrapped terminal 429 never advertises a longer wait than the pool's +// earliest auto-recovery. +func TestExecutePoolCooldownIncludesTransientNonCooldownBlock(t *testing.T) { + longQuotaCooldown := 3 * time.Hour + shortTransient := 90 * time.Second + + quotaAuth := newIssue7521Auth("7521-mixed-quota") + transientAuth := newIssue7521Auth("7521-mixed-transient") + execAuth := newIssue7521Auth("7521-mixed-exec") + + exec := &customStreamMockExecutor{ + identifier: "claude", + mockCustomErrorExecutor: mockCustomErrorExecutor{ + identifier: "claude", + executeFn: func(ctx context.Context, _ *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { + cliproxyexecutor.MarkUpstreamAttempt(ctx) + return cliproxyexecutor.Response{}, newIssue7521Upstream429("last credential rate limited", 2*time.Hour) + }, + }, + } + manager := newIssue7521Manager(t, nil, exec, quotaAuth, transientAuth, execAuth) + + // Credential A: genuine quota cooldown recovering in 3h. + manager.MarkResult(context.Background(), Result{ + AuthID: quotaAuth.ID, + Provider: "claude", + Model: issue7521PoolModel, + Success: false, + RetryAfter: &longQuotaCooldown, + CredentialScope: true, + Error: &Error{HTTPStatus: http.StatusTooManyRequests, Message: "quota exhausted"}, + }) + + // Credential B: transient non-cooldown block recovering in 90s. Recording + // a transient_rate_limit result (non-credential scope) marks only the + // reporting model unavailable without a quota label, so the scan + // classifies it as blockReasonOther rather than blockReasonCooldown. + manager.MarkResult(context.Background(), Result{ + AuthID: transientAuth.ID, + Provider: "claude", + Model: issue7521PoolModel, + Success: false, + RetryAfter: &shortTransient, + Error: &Error{ + Code: ErrorCodeTransientRateLimit, + HTTPStatus: http.StatusTooManyRequests, + Message: "transient capacity rejection", + }, + }) + + // Credential C is the only unblocked candidate; it fails and joins the + // pool cooldown. The pool's earliest recovery is B's +90s, not A's +3h nor + // the attempt's own 2h hint. + _, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{}) + assertIssue7521PoolCooldownError(t, errExec, "last credential rate limited", shortTransient) +} diff --git a/sdk/cliproxy/auth/conductor_selection.go b/sdk/cliproxy/auth/conductor_selection.go index ccc09fa541..fccb0fffdf 100644 --- a/sdk/cliproxy/auth/conductor_selection.go +++ b/sdk/cliproxy/auth/conductor_selection.go @@ -3012,6 +3012,144 @@ func authCoolingSummary(auth *Auth, model string, next time.Time, now time.Time) return fmt.Sprintf("[%s, reason=%s, remaining=%s]", ident, reason, remaining) } +func selectorExcludesZeroWeight(selector Selector) bool { + if _, weighted := selector.(*WeightedRoundRobinSelector); weighted { + return true + } + if affinity, ok := selector.(*SessionAffinitySelector); ok && affinity != nil { + _, weighted := affinity.fallback.(*WeightedRoundRobinSelector) + return weighted + } + return false +} + +// earliestPoolCooldownForModelAt scans the model's candidate credentials with +// the same filters selection uses. It reports: +// - earliest: the earliest active timed cooldown recovery instant +// - hasUnblocked: true when at least one eligible candidate is still available +// - ok: true when no candidate is available and at least one is in a timed cooldown +func (m *Manager) earliestPoolCooldownForModelAt(ctx context.Context, providers []string, model string, opts cliproxyexecutor.Options, now time.Time) (earliest time.Time, hasUnblocked bool, ok bool) { + if m == nil { + return time.Time{}, false, false + } + excludeZeroWeight := selectorExcludesZeroWeight(m.Selector()) + m.mu.RLock() + defer m.mu.RUnlock() + eligibility := authSelectionEligibilityForRequest(ctx, opts) + pinnedAuthID := pinnedAuthIDFromMetadata(opts.Metadata) + providerSet := make(map[string]struct{}, len(providers)) + for _, p := range providers { + if norm := canonicalSchedulingProvider(p); norm != "" && norm != "mixed" { + providerSet[norm] = struct{}{} + } + } + registryRef := registry.GetGlobalRegistry() + trimmedModel := strings.TrimSpace(model) + + totalCandidates := 0 + cooling := 0 + for _, candidate := range m.auths { + if candidate == nil || candidate.Disabled { + continue + } + if excludeZeroWeight && authWeight(candidate) <= 0 { + continue + } + rawProvider := executorKeyFromAuth(candidate) + providerKey := canonicalSchedulingProvider(rawProvider) + if providerKey == "" { + continue + } + if len(providerSet) > 0 { + if _, match := providerSet[providerKey]; !match { + continue + } + } + if _, hasExec := m.executorLocked(rawProvider); !hasExec { + if _, hasNormExec := m.executorLocked(providerKey); !hasNormExec { + continue + } + } + if pinnedAuthID != "" && candidate.ID != pinnedAuthID { + continue + } + if !eligibility.allows(candidate) { + continue + } + if trimmedModel != "" && !m.authSupportsRouteModel(registryRef, candidate, model) { + continue + } + totalCandidates++ + checkModel := m.selectionModelForAuth(candidate, model) + blocked, reason, next := isAuthBlockedForModelForRequest(ctx, candidate, checkModel, now, opts.Headers) + if !blocked { + return time.Time{}, true, false + } + if reason == blockReasonCooldown { + cooling++ + } + // Count every timed auto-recovery toward the pool's earliest reset, + // including transient non-cooldown blocks (e.g. a short capacity 429), + // matching how selection computes the earliest availability. + if reason != blockReasonDisabled && !next.IsZero() && next.After(now) { + if earliest.IsZero() || next.Before(earliest) { + earliest = next + } + } + } + if totalCandidates > 0 && cooling > 0 && !earliest.IsZero() { + return earliest, false, true + } + return time.Time{}, false, false +} + +func observeSelectionCooldownDeadline(dst *time.Time, err error, now time.Time) { + if dst == nil || err == nil { + return + } + var cooldown *modelCooldownError + if !errors.As(err, &cooldown) || cooldown == nil || cooldown.resetIn <= 0 { + return + } + candidate := now.Add(cooldown.resetIn) + if dst.IsZero() || candidate.Before(*dst) { + *dst = candidate + } +} + +func (m *Manager) annotatePoolCooldownTerminalError(ctx context.Context, providers []string, model string, opts cliproxyexecutor.Options, lastErr error, selectionCooldownDeadline time.Time) error { + return m.annotatePoolCooldownTerminalErrorAt(ctx, providers, model, opts, lastErr, selectionCooldownDeadline, time.Now()) +} + +func (m *Manager) annotatePoolCooldownTerminalErrorAt(ctx context.Context, providers []string, model string, opts cliproxyexecutor.Options, lastErr error, selectionCooldownDeadline time.Time, now time.Time) error { + if m == nil || lastErr == nil { + return lastErr + } + if statusCodeFromError(lastErr) != http.StatusTooManyRequests { + return lastErr + } + var marker interface{ IsModelCooldown() bool } + if errors.As(lastErr, &marker) && marker != nil && marker.IsModelCooldown() { + return lastErr + } + var existing *poolCooldownTerminalError + if errors.As(lastErr, &existing) && existing != nil { + return lastErr + } + deadline, hasUnblocked, ok := m.earliestPoolCooldownForModelAt(ctx, providers, model, opts, now) + if hasUnblocked { + return lastErr + } + if !ok && !selectionCooldownDeadline.IsZero() && selectionCooldownDeadline.After(now) { + deadline = selectionCooldownDeadline + ok = true + } + if !ok { + return lastErr + } + return &poolCooldownTerminalError{cause: lastErr, deadline: deadline} +} + func (m *Manager) warnLogAuthUnavailable(ctx context.Context, providers []string, model string, opts cliproxyexecutor.Options, tried map[string]struct{}, err error) { if m == nil || err == nil || !isAuthUnavailableError(err) { return diff --git a/sdk/cliproxy/auth/errors.go b/sdk/cliproxy/auth/errors.go index 9f94371485..640219c252 100644 --- a/sdk/cliproxy/auth/errors.go +++ b/sdk/cliproxy/auth/errors.go @@ -358,6 +358,87 @@ func (e *authUnavailableError) Headers() http.Header { return safeRetryAfterHeader(e.retryAfter) } +// poolCooldownTerminalError wraps a terminal upstream 429 when every eligible +// candidate credential for the requested model is blocked and at least one is +// in a timed cooldown. It keeps the upstream error text, body, status, and +// passthrough headers while marking the error as a model cooldown and exposing +// the pool's earliest recovery deadline via Retry-After. +type poolCooldownTerminalError struct { + cause error + deadline time.Time +} + +func (e *poolCooldownTerminalError) Error() string { + if e == nil || e.cause == nil { + return "" + } + return e.cause.Error() +} + +func (e *poolCooldownTerminalError) Unwrap() error { + if e == nil { + return nil + } + return e.cause +} + +func (e *poolCooldownTerminalError) As(target any) bool { + if e == nil { + return false + } + if t, ok := target.(**Error); ok { + var base *Error + if errors.As(e.cause, &base) && base != nil { + *t = base + return true + } + } + return false +} + +func (e *poolCooldownTerminalError) IsModelCooldown() bool { + return e != nil && !e.deadline.IsZero() +} + +func (e *poolCooldownTerminalError) StatusCode() int { + if e == nil { + return 0 + } + return statusCodeFromError(e.cause) +} + +func (e *poolCooldownTerminalError) RetryAfter() *time.Duration { + if e == nil || e.deadline.IsZero() { + return nil + } + remaining := time.Until(e.deadline) + if remaining <= 0 { + return nil + } + return &remaining +} + +func (e *poolCooldownTerminalError) SafeResponseHeaders() http.Header { + if e == nil || e.deadline.IsZero() { + return nil + } + return safeRetryAfterHeader(time.Until(e.deadline)) +} + +func (e *poolCooldownTerminalError) Headers() http.Header { + if e == nil || e.cause == nil { + return nil + } + for cur := e.cause; cur != nil; cur = errors.Unwrap(cur) { + if carrier, ok := cur.(interface{ Headers() http.Header }); ok && carrier != nil { + if hdr := carrier.Headers(); len(hdr) > 0 { + return hdr + } + } + } + return nil +} + // trustedRetryAfterError annotates a terminal upstream rejection whose // locally recorded cooldown deadline governs the client-visible Retry-After. // The wrapped error keeps its own status, body, identity and retry hint; the @@ -507,6 +588,10 @@ func HasTrustedRetryDeadline(err error) bool { if SafeResponseHeaders(err).Get("Retry-After") != "" { return true } + var poolCooldown *poolCooldownTerminalError + if errors.As(err, &poolCooldown) && poolCooldown != nil && !poolCooldown.deadline.IsZero() { + return true + } var trusted *trustedRetryAfterError return errors.As(err, &trusted) && trusted != nil && !trusted.deadline.IsZero() } @@ -518,6 +603,12 @@ func trustedRetryAfterFromError(err error) *time.Duration { if err == nil { return nil } + var poolCooldown *poolCooldownTerminalError + if errors.As(err, &poolCooldown) && poolCooldown != nil { + if remaining := poolCooldown.RetryAfter(); remaining != nil && *remaining > 0 { + return remaining + } + } var trusted *trustedRetryAfterError if errors.As(err, &trusted) && trusted != nil { if remaining := trusted.RetryAfter(); remaining != nil && *remaining > 0 { diff --git a/sdk/cliproxy/auth/home_concurrency.go b/sdk/cliproxy/auth/home_concurrency.go index 8a3f47a4db..9c3082b6b9 100644 --- a/sdk/cliproxy/auth/home_concurrency.go +++ b/sdk/cliproxy/auth/home_concurrency.go @@ -292,6 +292,10 @@ func SafeResponseHeaders(err error) http.Header { } return nil } + var poolCooldown *poolCooldownTerminalError + if errors.As(err, &poolCooldown) && poolCooldown != nil { + return poolCooldown.SafeResponseHeaders() + } var trusted *trustedRetryAfterError if errors.As(err, &trusted) && trusted != nil { return trusted.SafeResponseHeaders()