fix(auth): carry the pool cooldown reset on the terminal 429

- Wrap the final 429 in a poolCooldownTerminalError at the Execute/ExecuteCount/ExecuteStream boundaries when every candidate credential for the model is blocked and at least one is in a timed cooldown, preserving the upstream error body and status while marking IsModelCooldown so the Claude handler appends the reset time.
- Compute the Retry-After from the earliest auto-recovery across all blocked candidates (including non-cooldown timed blocks) so the client sees the pool's real recovery instead of the last attempt's hint.
- Add regression coverage for both cooldown variants across the three execution paths, the available-candidate guard, mixed-block earliest deadlines, the sub-floor hint clamp, and the Claude handler end-to-end.

Closes: #7521
This commit is contained in:
Luis Pater
2026-11-14 03:53:41 +08:00
parent 100f2de1a5
commit 4da9f68527
7 changed files with 881 additions and 1 deletions

View File

@@ -0,0 +1,174 @@
package claude
import (
"context"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"testing"
"time"
"github.com/MoeForge/moego/ojson"
"github.com/gin-gonic/gin"
"github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
"github.com/router-for-me/CLIProxyAPI/v8/sdk/api/handlers"
coreauth "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/auth"
coreexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
sdkconfig "github.com/router-for-me/CLIProxyAPI/v8/sdk/config"
)
type issue7521UpstreamError struct {
status int
body string
retryAfter time.Duration
headers http.Header
}
func (e *issue7521UpstreamError) Error() string { return e.body }
func (e *issue7521UpstreamError) StatusCode() int { return e.status }
func (e *issue7521UpstreamError) RetryAfter() *time.Duration {
d := e.retryAfter
return &d
}
func (e *issue7521UpstreamError) Headers() http.Header {
if e.headers == nil {
return nil
}
return e.headers.Clone()
}
type issue7521ClaudeExecutor struct {
executeFn func(ctx context.Context, auth *coreauth.Auth, req coreexecutor.Request, opts coreexecutor.Options) (coreexecutor.Response, error)
streamFn func(ctx context.Context, auth *coreauth.Auth, req coreexecutor.Request, opts coreexecutor.Options) (*coreexecutor.StreamResult, error)
countFn func(ctx context.Context, auth *coreauth.Auth, req coreexecutor.Request, opts coreexecutor.Options) (coreexecutor.Response, error)
}
func (e *issue7521ClaudeExecutor) Identifier() string { return "claude" }
func (e *issue7521ClaudeExecutor) Execute(ctx context.Context, auth *coreauth.Auth, req coreexecutor.Request, opts coreexecutor.Options) (coreexecutor.Response, error) {
coreexecutor.MarkUpstreamAttempt(ctx)
return e.executeFn(ctx, auth, req, opts)
}
func (e *issue7521ClaudeExecutor) ExecuteStream(ctx context.Context, auth *coreauth.Auth, req coreexecutor.Request, opts coreexecutor.Options) (*coreexecutor.StreamResult, error) {
coreexecutor.MarkUpstreamAttempt(ctx)
return e.streamFn(ctx, auth, req, opts)
}
func (e *issue7521ClaudeExecutor) CountTokens(ctx context.Context, auth *coreauth.Auth, req coreexecutor.Request, opts coreexecutor.Options) (coreexecutor.Response, error) {
coreexecutor.MarkUpstreamAttempt(ctx)
return e.countFn(ctx, auth, req, opts)
}
func (e *issue7521ClaudeExecutor) Refresh(_ context.Context, auth *coreauth.Auth) (*coreauth.Auth, error) {
return auth, nil
}
func (e *issue7521ClaudeExecutor) HttpRequest(_ context.Context, _ *coreauth.Auth, _ *http.Request) (*http.Response, error) {
return nil, nil
}
func TestClaudeLastCredential429AppendsPoolResetTime(t *testing.T) {
gin.SetMode(gin.TestMode)
const model = "issue-7521-claude-route-model"
shortHint := 95 * time.Minute
longHint := 3 * time.Hour
makeErr := func(authID string) error {
if strings.HasSuffix(authID, "-a") {
return &issue7521UpstreamError{
status: http.StatusTooManyRequests,
body: `{"type":"error","error":{"type":"rate_limit_error","message":"First account rate limited."}}`,
retryAfter: shortHint,
headers: http.Header{"Retry-After": {"5700"}},
}
}
return &issue7521UpstreamError{
status: http.StatusTooManyRequests,
body: `{"type":"error","error":{"type":"rate_limit_error","message":"This request would exceed your account's monthly spend limit. Please try again later."}}`,
retryAfter: longHint,
headers: http.Header{"Retry-After": {"10800"}, "X-Upstream-Trace": {"trace-b"}},
}
}
for _, tc := range []struct {
name string
path string
streaming bool
}{
{name: "messages_non_stream", path: "/v1/messages", streaming: false},
{name: "messages_stream", path: "/v1/messages", streaming: true},
{name: "count_tokens", path: "/v1/messages/count_tokens", streaming: false},
} {
t.Run(tc.name, func(t *testing.T) {
manager := coreauth.NewManager(nil, nil, nil)
manager.RegisterExecutor(&issue7521ClaudeExecutor{
executeFn: func(_ context.Context, auth *coreauth.Auth, _ coreexecutor.Request, _ coreexecutor.Options) (coreexecutor.Response, error) {
return coreexecutor.Response{}, makeErr(auth.ID)
},
streamFn: func(_ context.Context, auth *coreauth.Auth, _ coreexecutor.Request, _ coreexecutor.Options) (*coreexecutor.StreamResult, error) {
chunks := make(chan coreexecutor.StreamChunk, 1)
chunks <- coreexecutor.StreamChunk{Err: makeErr(auth.ID)}
close(chunks)
return &coreexecutor.StreamResult{Chunks: chunks}, nil
},
countFn: func(_ context.Context, auth *coreauth.Auth, _ coreexecutor.Request, _ coreexecutor.Options) (coreexecutor.Response, error) {
return coreexecutor.Response{}, makeErr(auth.ID)
},
})
reg := registry.GetGlobalRegistry()
for _, suffix := range []string{"-a", "-b"} {
authID := "issue-7521-route-" + tc.name + suffix
auth := &coreauth.Auth{
ID: authID,
Provider: "claude",
Status: coreauth.StatusActive,
Attributes: map[string]string{"api_key": "key" + suffix},
}
if _, errReg := manager.Register(context.Background(), auth); errReg != nil {
t.Fatalf("register auth: %v", errReg)
}
reg.RegisterClient(authID, "claude", []*registry.ModelInfo{{ID: model}})
t.Cleanup(func() {
reg.UnregisterClient(authID)
})
}
handler := NewClaudeCodeAPIHandler(handlers.NewBaseAPIHandlers(&sdkconfig.SDKConfig{PassthroughHeaders: true}, manager))
router := gin.New()
router.POST("/v1/messages", handler.ClaudeMessages)
router.POST("/v1/messages/count_tokens", handler.ClaudeCountTokens)
body := `{"model":"` + model + `","messages":[{"role":"user","content":"hi"}],"stream":` + strconv.FormatBool(tc.streaming) + `}`
req := httptest.NewRequest(http.MethodPost, tc.path, strings.NewReader(body))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusTooManyRequests {
t.Fatalf("status = %d, want 429; body=%s", rec.Code, rec.Body.String())
}
seconds, errParse := strconv.ParseInt(rec.Header().Get("Retry-After"), 10, 64)
if errParse != nil || seconds <= 0 {
t.Fatalf("Retry-After = %q, want positive integer seconds", rec.Header().Get("Retry-After"))
}
if seconds > int64((shortHint + 10*time.Second).Seconds()) {
t.Fatalf("Retry-After = %d, want pool deadline near %d rather than last attempt 10800", seconds, int64(shortHint.Seconds()))
}
if gotTrace := rec.Header().Get("X-Upstream-Trace"); gotTrace != "trace-b" {
t.Fatalf("X-Upstream-Trace = %q, want trace-b preserved under passthrough-headers", gotTrace)
}
if gotType := ojson.GetBytesNoCopy(rec.Body.Bytes(), "error.type").String(); gotType != "rate_limit_error" {
t.Fatalf("error.type = %q, want rate_limit_error", gotType)
}
wantDelay := (time.Duration(seconds) * time.Second).String()
wantMessage := "This request would exceed your account's monthly spend limit. Please try again later. (reset after " + wantDelay + ")"
if gotMsg := ojson.GetBytesNoCopy(rec.Body.Bytes(), "error.message").String(); gotMsg != wantMessage {
t.Fatalf("error.message = %q, want %q", gotMsg, wantMessage)
}
})
}
}

View File

@@ -3455,7 +3455,8 @@ func retryAfterFromError(err error) *time.Duration {
// An expired trusted wrapper reports a negative remaining time; natural
// expiry is not an invalid provider hint, so fall through to the wrapped
// error's own hint instead of surfacing the negative value.
if _, isTrusted := rap.(*trustedRetryAfterError); isTrusted {
switch rap.(type) {
case *poolCooldownTerminalError, *trustedRetryAfterError:
if inner := errors.Unwrap(err); inner != nil {
return retryAfterFromError(inner)
}

View File

@@ -224,6 +224,7 @@ func (m *Manager) Execute(ctx context.Context, providers []string, req cliproxye
var lastErr error
var preferredUpstreamErr error
var selectionCooldownDeadline time.Time
var terminalRoundRejectedByScheduler bool
retryModel := authSelectionModelFromOptions(opts, req.Model)
for attempt := 0; ; attempt++ {
@@ -244,6 +245,7 @@ func (m *Manager) Execute(ctx context.Context, providers []string, req cliproxye
break
}
terminalRoundRejectedByScheduler = hasRoundEndedBySchedulerReject(errExec)
observeSelectionCooldownDeadline(&selectionCooldownDeadline, errExec, time.Now())
if hasUpstreamExecutionAttempt(errExec) {
preferredUpstreamErr = errExec
}
@@ -272,6 +274,7 @@ func (m *Manager) Execute(ctx context.Context, providers []string, req cliproxye
return resp, nil
}
}
lastErr = m.annotatePoolCooldownTerminalError(ctx, normalized, retryModel, opts, lastErr, selectionCooldownDeadline)
return cliproxyexecutor.Response{}, lastErr
}
return cliproxyexecutor.Response{}, &Error{Code: "auth_not_found", Message: "no auth available"}
@@ -299,6 +302,7 @@ func (m *Manager) ExecuteCount(ctx context.Context, providers []string, req clip
var lastErr error
var preferredUpstreamErr error
var selectionCooldownDeadline time.Time
retryModel := authSelectionModelFromOptions(opts, req.Model)
for attempt := 0; ; attempt++ {
roundAttempted := make(map[string]struct{})
@@ -316,6 +320,7 @@ func (m *Manager) ExecuteCount(ctx context.Context, providers []string, req clip
}
break
}
observeSelectionCooldownDeadline(&selectionCooldownDeadline, errExec, time.Now())
if hasUpstreamExecutionAttempt(errExec) {
preferredUpstreamErr = errExec
}
@@ -337,6 +342,7 @@ func (m *Manager) ExecuteCount(ctx context.Context, providers []string, req clip
lastErr = preferredExecutionAttemptError(lastErr, preferredUpstreamErr)
lastErr = unwrapExecutionBoundaryError(lastErr)
lastErr = m.normalizeSessionAffinityNeverTerminalError(lastErr, normalized, retryModel, opts)
lastErr = m.annotatePoolCooldownTerminalError(ctx, normalized, retryModel, opts, lastErr, selectionCooldownDeadline)
return cliproxyexecutor.Response{}, lastErr
}
return cliproxyexecutor.Response{}, &Error{Code: "auth_not_found", Message: "no auth available"}
@@ -375,6 +381,7 @@ func (m *Manager) ExecuteStream(ctx context.Context, providers []string, req cli
var lastErr error
var preferredUpstreamErr error
var selectionCooldownDeadline time.Time
var terminalRoundRejectedByScheduler bool
homeRetryLimit := -1
retryModel := authSelectionModelFromOptions(opts, req.Model)
@@ -388,6 +395,9 @@ func (m *Manager) ExecuteStream(ctx context.Context, providers []string, req cli
if errStream == nil {
return result, nil
}
if !m.HomeEnabled() {
observeSelectionCooldownDeadline(&selectionCooldownDeadline, errStream, time.Now())
}
if hasUpstreamExecutionAttempt(errStream) {
preferredUpstreamErr = errStream
}
@@ -446,6 +456,9 @@ func (m *Manager) ExecuteStream(ctx context.Context, providers []string, req cli
return result, nil
}
}
if !m.HomeEnabled() {
lastErr = m.annotatePoolCooldownTerminalError(ctx, normalized, retryModel, opts, lastErr, selectionCooldownDeadline)
}
var bootstrapErr *streamBootstrapError
if errors.As(lastErr, &bootstrapErr) && bootstrapErr != nil {
return streamErrorResult(bootstrapErr.Headers(), lastErr), nil

View File

@@ -0,0 +1,459 @@
package auth
import (
"context"
"errors"
"net/http"
"strconv"
"strings"
"testing"
"time"
"github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
)
const issue7521PoolModel = "claude-sonnet-4-6-issue-7521"
func newIssue7521Manager(
t *testing.T,
selector Selector,
exec ProviderExecutor,
auths ...*Auth,
) *Manager {
t.Helper()
withQuotaCooldownEnabled(t)
if selector == nil {
selector = &RoundRobinSelector{}
}
manager := NewManager(nil, selector, nil)
manager.SetRetryConfig(2, 30*time.Second, 0)
manager.RegisterExecutor(exec)
reg := registry.GetGlobalRegistry()
for _, auth := range auths {
authID := auth.ID
reg.RegisterClient(authID, auth.Provider, []*registry.ModelInfo{{ID: issue7521PoolModel}})
t.Cleanup(func() {
reg.UnregisterClient(authID)
})
if _, errRegister := manager.Register(context.Background(), auth); errRegister != nil {
t.Fatalf("register auth %s: %v", authID, errRegister)
}
}
return manager
}
func newIssue7521Auth(id string) *Auth {
return &Auth{
ID: id,
Provider: "claude",
Status: StatusActive,
Attributes: map[string]string{"api_key": "key-" + id},
}
}
func newIssue7521Upstream429(msg string, retryAfter time.Duration) error {
hint := retryAfter
return customStatusError{
code: http.StatusTooManyRequests,
msg: msg,
retryAfter: &hint,
}
}
func assertIssue7521PoolCooldownError(t *testing.T, err error, wantMessage string, wantDeadline time.Duration) {
t.Helper()
if err == nil {
t.Fatal("expected terminal 429 error, got nil")
}
if got := statusCodeFromError(err); got != http.StatusTooManyRequests {
t.Fatalf("status = %d, want 429", got)
}
if !strings.Contains(err.Error(), wantMessage) {
t.Fatalf("error = %q, want upstream message %q preserved", err.Error(), wantMessage)
}
var marker interface{ IsModelCooldown() bool }
if !errors.As(err, &marker) || marker == nil || !marker.IsModelCooldown() {
t.Fatalf("error %T (%v) must report IsModelCooldown() == true", err, err)
}
if !HasTrustedRetryDeadline(err) {
t.Fatal("HasTrustedRetryDeadline(err) = false, want true")
}
retryAfter := SafeResponseHeaders(err).Get("Retry-After")
seconds, errParse := strconv.ParseInt(retryAfter, 10, 64)
if errParse != nil || seconds <= 0 {
t.Fatalf("Retry-After = %q, want positive integer seconds", retryAfter)
}
gotDuration := time.Duration(seconds) * time.Second
if gotDuration > wantDeadline+10*time.Second || gotDuration < wantDeadline-10*time.Second {
t.Fatalf("Retry-After = %ds (%s), want near pool recovery %s", seconds, gotDuration, wantDeadline)
}
}
// Variant 2: fresh start where every candidate credential is tried in the same
// round and returns 429; no modelCooldownError is built during selection, yet
// the terminal 429 must report IsModelCooldown() and carry the earliest pool reset.
func TestExecuteLastCredential429CarriesEarliestPoolReset(t *testing.T) {
first := newIssue7521Auth("7521-exec-a")
second := newIssue7521Auth("7521-exec-b")
shortHint := 90 * time.Minute
longHint := 3 * time.Hour
var attempts []string
exec := &customStreamMockExecutor{
identifier: "claude",
mockCustomErrorExecutor: mockCustomErrorExecutor{
identifier: "claude",
executeFn: func(ctx context.Context, auth *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
cliproxyexecutor.MarkUpstreamAttempt(ctx)
attempts = append(attempts, auth.ID)
if auth.ID == first.ID {
return cliproxyexecutor.Response{}, newIssue7521Upstream429("first credential rate limited", shortHint)
}
return cliproxyexecutor.Response{}, newIssue7521Upstream429("monthly spend limit reached", longHint)
},
},
}
manager := newIssue7521Manager(t, nil, exec, first, second)
_, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{})
assertIssue7521PoolCooldownError(t, errExec, "monthly spend limit reached", shortHint)
if len(attempts) != 2 {
t.Fatalf("attempts = %v, want 2 credentials tried", attempts)
}
}
func TestExecuteSingleCredentialSubFloorHintReportsActualPoolRecovery(t *testing.T) {
single := newIssue7521Auth("7521-subfloor-a")
hint := 7 * time.Second
exec := &customStreamMockExecutor{
identifier: "claude",
mockCustomErrorExecutor: mockCustomErrorExecutor{
identifier: "claude",
executeFn: func(ctx context.Context, _ *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
cliproxyexecutor.MarkUpstreamAttempt(ctx)
return cliproxyexecutor.Response{}, newIssue7521Upstream429("try later", hint)
},
},
}
manager := newIssue7521Manager(t, nil, exec, single)
manager.SetRetryConfig(0, 0, 0)
_, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{})
if errExec == nil {
t.Fatal("expected terminal 429 error")
}
// MarkResult clamps the 7s hint up to minQuotaCooldownFloor (10s), so the
// pool's actual recovery is 10s out. The wrap must advertise that real
// deadline, not the shorter per-attempt hint.
if got := SafeResponseHeaders(errExec).Get("Retry-After"); got != "10" {
t.Fatalf("Retry-After = %q, want 10 (minQuotaCooldownFloor)", got)
}
var marker interface{ IsModelCooldown() bool }
if !errors.As(errExec, &marker) || marker == nil || !marker.IsModelCooldown() {
t.Fatalf("error %T must report IsModelCooldown() == true", errExec)
}
}
// Variant 1: one credential is already cooling down; the remaining credential
// returns 429 and the next pick fails with modelCooldownError, which is
// replaced by preferredExecutionAttemptError. The terminal 429 must still keep
// the upstream text and carry the earliest pool cooldown reset.
func TestExecuteLastUsableCredential429KeepsExistingCooldownReset(t *testing.T) {
first := newIssue7521Auth("7521-var1-a")
second := newIssue7521Auth("7521-var1-b")
existingCooldown := 75 * time.Minute
freshHint := 3 * time.Hour
exec := &customStreamMockExecutor{
identifier: "claude",
mockCustomErrorExecutor: mockCustomErrorExecutor{
identifier: "claude",
executeFn: func(ctx context.Context, _ *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
cliproxyexecutor.MarkUpstreamAttempt(ctx)
return cliproxyexecutor.Response{}, newIssue7521Upstream429("second credential rate limited", freshHint)
},
},
}
manager := newIssue7521Manager(t, nil, exec, first, second)
manager.MarkResult(context.Background(), Result{
AuthID: first.ID,
Provider: "claude",
Model: issue7521PoolModel,
Success: false,
RetryAfter: &existingCooldown,
CredentialScope: true,
Error: &Error{HTTPStatus: http.StatusTooManyRequests, Message: "already cooling"},
})
_, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{})
assertIssue7521PoolCooldownError(t, errExec, "second credential rate limited", existingCooldown)
}
func TestExecuteCountLastCredential429CarriesEarliestPoolReset(t *testing.T) {
first := newIssue7521Auth("7521-count-a")
second := newIssue7521Auth("7521-count-b")
shortHint := 85 * time.Minute
longHint := 4 * time.Hour
exec := &customStreamMockExecutor{
identifier: "claude",
mockCustomErrorExecutor: mockCustomErrorExecutor{
identifier: "claude",
countFn: func(ctx context.Context, auth *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
cliproxyexecutor.MarkUpstreamAttempt(ctx)
if auth.ID == first.ID {
return cliproxyexecutor.Response{}, newIssue7521Upstream429("count first limited", shortHint)
}
return cliproxyexecutor.Response{}, newIssue7521Upstream429("count second limited", longHint)
},
},
}
manager := newIssue7521Manager(t, nil, exec, first, second)
_, errCount := manager.ExecuteCount(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{})
assertIssue7521PoolCooldownError(t, errCount, "count second limited", shortHint)
}
func TestExecuteStreamLastCredential429CarriesEarliestPoolReset(t *testing.T) {
shortHint := 80 * time.Minute
longHint := 3 * time.Hour
t.Run("direct stream error", func(t *testing.T) {
first := newIssue7521Auth("7521-stream-direct-a")
second := newIssue7521Auth("7521-stream-direct-b")
exec := &customStreamMockExecutor{
identifier: "claude",
streamFn: func(ctx context.Context, auth *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) {
cliproxyexecutor.MarkUpstreamAttempt(ctx)
if auth.ID == first.ID {
return nil, newIssue7521Upstream429("stream first limited", shortHint)
}
return nil, newIssue7521Upstream429("stream second limited", longHint)
},
}
manager := newIssue7521Manager(t, nil, exec, first, second)
_, errStream := manager.ExecuteStream(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{Stream: true})
assertIssue7521PoolCooldownError(t, errStream, "stream second limited", shortHint)
})
t.Run("bootstrap chunk error", func(t *testing.T) {
first := newIssue7521Auth("7521-stream-boot-a")
second := newIssue7521Auth("7521-stream-boot-b")
exec := &customStreamMockExecutor{
identifier: "claude",
streamFn: func(ctx context.Context, auth *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) {
cliproxyexecutor.MarkUpstreamAttempt(ctx)
chunks := make(chan cliproxyexecutor.StreamChunk, 1)
if auth.ID == first.ID {
chunks <- cliproxyexecutor.StreamChunk{Err: newIssue7521Upstream429("bootstrap first limited", shortHint)}
} else {
chunks <- cliproxyexecutor.StreamChunk{Err: newIssue7521Upstream429("bootstrap second limited", longHint)}
}
close(chunks)
return &cliproxyexecutor.StreamResult{Chunks: chunks}, nil
},
}
manager := newIssue7521Manager(t, nil, exec, first, second)
streamRes, errStream := manager.ExecuteStream(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{Stream: true})
if errStream != nil {
t.Fatalf("expected streamErrorResult, got direct err: %v", errStream)
}
chunk, ok := <-streamRes.Chunks
if !ok {
t.Fatal("expected error chunk on bootstrap failure")
}
assertIssue7521PoolCooldownError(t, chunk.Err, "bootstrap second limited", shortHint)
})
}
func TestExecute429LeavesOrdinary429UnchangedWhenCandidateAvailable(t *testing.T) {
first := newIssue7521Auth("7521-avail-a")
second := newIssue7521Auth("7521-avail-b")
hint := time.Hour
exec := &customStreamMockExecutor{
identifier: "claude",
mockCustomErrorExecutor: mockCustomErrorExecutor{
identifier: "claude",
executeFn: func(ctx context.Context, _ *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
cliproxyexecutor.MarkUpstreamAttempt(ctx)
return cliproxyexecutor.Response{}, newIssue7521Upstream429("single attempt 429", hint)
},
},
}
manager := newIssue7521Manager(t, nil, exec, first, second)
// Stop after 1 credential attempt with 0 retries so the second credential remains unblocked.
manager.SetRetryConfig(0, 0, 1)
_, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{})
if errExec == nil {
t.Fatal("expected 429 error")
}
if got := statusCodeFromError(errExec); got != http.StatusTooManyRequests {
t.Fatalf("status = %d, want 429", got)
}
var marker interface{ IsModelCooldown() bool }
if errors.As(errExec, &marker) && marker != nil && marker.IsModelCooldown() {
t.Fatalf("ordinary 429 must not report IsModelCooldown() when another candidate is still available: %T (%v)", errExec, errExec)
}
}
func TestExecuteLastCredential429IgnoresZeroWeightCandidateUnderWeightedSelector(t *testing.T) {
active := newIssue7521Auth("7521-weight-active")
drained := newIssue7521Auth("7521-weight-zero")
drained.Attributes["weight"] = "0"
hint := 70 * time.Minute
exec := &customStreamMockExecutor{
identifier: "claude",
mockCustomErrorExecutor: mockCustomErrorExecutor{
identifier: "claude",
executeFn: func(ctx context.Context, _ *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
cliproxyexecutor.MarkUpstreamAttempt(ctx)
return cliproxyexecutor.Response{}, newIssue7521Upstream429("active credential exhausted", hint)
},
},
}
selector := NewSessionAffinitySelector(&WeightedRoundRobinSelector{})
manager := newIssue7521Manager(t, selector, exec, active, drained)
_, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{})
assertIssue7521PoolCooldownError(t, errExec, "active credential exhausted", hint)
}
func TestAnnotatePoolCooldownSelectionFallbackAndUnblockedGuard(t *testing.T) {
now := time.Date(2026, 11, 14, 12, 0, 0, 0, time.UTC)
fallbackDeadline := now.Add(42 * time.Minute)
lastAttemptHint := 3 * time.Hour
baseErr := withTrustedRetryAfter(newIssue7521Upstream429("upstream rate limited", lastAttemptHint), &lastAttemptHint)
t.Run("falls back to selection cooldown when no candidate unblocked", func(t *testing.T) {
manager := NewManager(nil, nil, nil)
wrapped := manager.annotatePoolCooldownTerminalErrorAt(
context.Background(),
[]string{"claude"},
issue7521PoolModel,
cliproxyexecutor.Options{},
baseErr,
fallbackDeadline,
now,
)
poolErr, ok := wrapped.(*poolCooldownTerminalError)
if !ok || poolErr == nil {
t.Fatalf("wrapped = %T, want *poolCooldownTerminalError", wrapped)
}
if !poolErr.deadline.Equal(fallbackDeadline) {
t.Fatalf("deadline = %v, want %v", poolErr.deadline, fallbackDeadline)
}
if !poolErr.IsModelCooldown() {
t.Fatal("IsModelCooldown() = false, want true")
}
})
t.Run("does not fall back when an eligible candidate is unblocked", func(t *testing.T) {
avail := newIssue7521Auth("7521-fallback-avail")
exec := &customStreamMockExecutor{identifier: "claude"}
manager := newIssue7521Manager(t, nil, exec, avail)
got := manager.annotatePoolCooldownTerminalErrorAt(
context.Background(),
[]string{"claude"},
issue7521PoolModel,
cliproxyexecutor.Options{},
baseErr,
fallbackDeadline,
now,
)
var marker interface{ IsModelCooldown() bool }
if errors.As(got, &marker) && marker != nil && marker.IsModelCooldown() {
t.Fatalf("must not wrap as model cooldown when a candidate is unblocked: %T", got)
}
})
t.Run("expired pool deadline suppresses raw header and falls back for retryAfterFromError", func(t *testing.T) {
cause := headerCarryingError{headers: http.Header{"X-Trace": {"1"}, "Retry-After": {"999"}}}
expired := &poolCooldownTerminalError{
cause: cause,
deadline: time.Now().Add(-time.Second),
}
if !HasTrustedRetryDeadline(expired) {
t.Fatal("HasTrustedRetryDeadline(expired) = false, want true")
}
if got := SafeResponseHeaders(expired).Get("Retry-After"); got != "" {
t.Fatalf("SafeResponseHeaders(expired) Retry-After = %q, want empty", got)
}
if got := expired.Headers().Get("X-Trace"); got != "1" {
t.Fatalf("Headers() X-Trace = %q, want 1", got)
}
agg := 2 * time.Second
homeTerminal := markHomeRetryRoundExhausted(&poolCooldownTerminalError{
cause: cause,
deadline: time.Now().Add(time.Hour),
}, &agg, false)
if got := SafeResponseHeaders(homeTerminal).Get("Retry-After"); got != "2" {
t.Fatalf("Home aggregate Retry-After = %q, want 2", got)
}
})
}
// Regression for the review finding: a short transient non-cooldown block on a
// sibling credential must pull the pool reset forward instead of being ignored,
// so the wrapped terminal 429 never advertises a longer wait than the pool's
// earliest auto-recovery.
func TestExecutePoolCooldownIncludesTransientNonCooldownBlock(t *testing.T) {
longQuotaCooldown := 3 * time.Hour
shortTransient := 90 * time.Second
quotaAuth := newIssue7521Auth("7521-mixed-quota")
transientAuth := newIssue7521Auth("7521-mixed-transient")
execAuth := newIssue7521Auth("7521-mixed-exec")
exec := &customStreamMockExecutor{
identifier: "claude",
mockCustomErrorExecutor: mockCustomErrorExecutor{
identifier: "claude",
executeFn: func(ctx context.Context, _ *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
cliproxyexecutor.MarkUpstreamAttempt(ctx)
return cliproxyexecutor.Response{}, newIssue7521Upstream429("last credential rate limited", 2*time.Hour)
},
},
}
manager := newIssue7521Manager(t, nil, exec, quotaAuth, transientAuth, execAuth)
// Credential A: genuine quota cooldown recovering in 3h.
manager.MarkResult(context.Background(), Result{
AuthID: quotaAuth.ID,
Provider: "claude",
Model: issue7521PoolModel,
Success: false,
RetryAfter: &longQuotaCooldown,
CredentialScope: true,
Error: &Error{HTTPStatus: http.StatusTooManyRequests, Message: "quota exhausted"},
})
// Credential B: transient non-cooldown block recovering in 90s. Recording
// a transient_rate_limit result (non-credential scope) marks only the
// reporting model unavailable without a quota label, so the scan
// classifies it as blockReasonOther rather than blockReasonCooldown.
manager.MarkResult(context.Background(), Result{
AuthID: transientAuth.ID,
Provider: "claude",
Model: issue7521PoolModel,
Success: false,
RetryAfter: &shortTransient,
Error: &Error{
Code: ErrorCodeTransientRateLimit,
HTTPStatus: http.StatusTooManyRequests,
Message: "transient capacity rejection",
},
})
// Credential C is the only unblocked candidate; it fails and joins the
// pool cooldown. The pool's earliest recovery is B's +90s, not A's +3h nor
// the attempt's own 2h hint.
_, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{})
assertIssue7521PoolCooldownError(t, errExec, "last credential rate limited", shortTransient)
}

View File

@@ -3012,6 +3012,144 @@ func authCoolingSummary(auth *Auth, model string, next time.Time, now time.Time)
return fmt.Sprintf("[%s, reason=%s, remaining=%s]", ident, reason, remaining)
}
func selectorExcludesZeroWeight(selector Selector) bool {
if _, weighted := selector.(*WeightedRoundRobinSelector); weighted {
return true
}
if affinity, ok := selector.(*SessionAffinitySelector); ok && affinity != nil {
_, weighted := affinity.fallback.(*WeightedRoundRobinSelector)
return weighted
}
return false
}
// earliestPoolCooldownForModelAt scans the model's candidate credentials with
// the same filters selection uses. It reports:
// - earliest: the earliest active timed cooldown recovery instant
// - hasUnblocked: true when at least one eligible candidate is still available
// - ok: true when no candidate is available and at least one is in a timed cooldown
func (m *Manager) earliestPoolCooldownForModelAt(ctx context.Context, providers []string, model string, opts cliproxyexecutor.Options, now time.Time) (earliest time.Time, hasUnblocked bool, ok bool) {
if m == nil {
return time.Time{}, false, false
}
excludeZeroWeight := selectorExcludesZeroWeight(m.Selector())
m.mu.RLock()
defer m.mu.RUnlock()
eligibility := authSelectionEligibilityForRequest(ctx, opts)
pinnedAuthID := pinnedAuthIDFromMetadata(opts.Metadata)
providerSet := make(map[string]struct{}, len(providers))
for _, p := range providers {
if norm := canonicalSchedulingProvider(p); norm != "" && norm != "mixed" {
providerSet[norm] = struct{}{}
}
}
registryRef := registry.GetGlobalRegistry()
trimmedModel := strings.TrimSpace(model)
totalCandidates := 0
cooling := 0
for _, candidate := range m.auths {
if candidate == nil || candidate.Disabled {
continue
}
if excludeZeroWeight && authWeight(candidate) <= 0 {
continue
}
rawProvider := executorKeyFromAuth(candidate)
providerKey := canonicalSchedulingProvider(rawProvider)
if providerKey == "" {
continue
}
if len(providerSet) > 0 {
if _, match := providerSet[providerKey]; !match {
continue
}
}
if _, hasExec := m.executorLocked(rawProvider); !hasExec {
if _, hasNormExec := m.executorLocked(providerKey); !hasNormExec {
continue
}
}
if pinnedAuthID != "" && candidate.ID != pinnedAuthID {
continue
}
if !eligibility.allows(candidate) {
continue
}
if trimmedModel != "" && !m.authSupportsRouteModel(registryRef, candidate, model) {
continue
}
totalCandidates++
checkModel := m.selectionModelForAuth(candidate, model)
blocked, reason, next := isAuthBlockedForModelForRequest(ctx, candidate, checkModel, now, opts.Headers)
if !blocked {
return time.Time{}, true, false
}
if reason == blockReasonCooldown {
cooling++
}
// Count every timed auto-recovery toward the pool's earliest reset,
// including transient non-cooldown blocks (e.g. a short capacity 429),
// matching how selection computes the earliest availability.
if reason != blockReasonDisabled && !next.IsZero() && next.After(now) {
if earliest.IsZero() || next.Before(earliest) {
earliest = next
}
}
}
if totalCandidates > 0 && cooling > 0 && !earliest.IsZero() {
return earliest, false, true
}
return time.Time{}, false, false
}
func observeSelectionCooldownDeadline(dst *time.Time, err error, now time.Time) {
if dst == nil || err == nil {
return
}
var cooldown *modelCooldownError
if !errors.As(err, &cooldown) || cooldown == nil || cooldown.resetIn <= 0 {
return
}
candidate := now.Add(cooldown.resetIn)
if dst.IsZero() || candidate.Before(*dst) {
*dst = candidate
}
}
func (m *Manager) annotatePoolCooldownTerminalError(ctx context.Context, providers []string, model string, opts cliproxyexecutor.Options, lastErr error, selectionCooldownDeadline time.Time) error {
return m.annotatePoolCooldownTerminalErrorAt(ctx, providers, model, opts, lastErr, selectionCooldownDeadline, time.Now())
}
func (m *Manager) annotatePoolCooldownTerminalErrorAt(ctx context.Context, providers []string, model string, opts cliproxyexecutor.Options, lastErr error, selectionCooldownDeadline time.Time, now time.Time) error {
if m == nil || lastErr == nil {
return lastErr
}
if statusCodeFromError(lastErr) != http.StatusTooManyRequests {
return lastErr
}
var marker interface{ IsModelCooldown() bool }
if errors.As(lastErr, &marker) && marker != nil && marker.IsModelCooldown() {
return lastErr
}
var existing *poolCooldownTerminalError
if errors.As(lastErr, &existing) && existing != nil {
return lastErr
}
deadline, hasUnblocked, ok := m.earliestPoolCooldownForModelAt(ctx, providers, model, opts, now)
if hasUnblocked {
return lastErr
}
if !ok && !selectionCooldownDeadline.IsZero() && selectionCooldownDeadline.After(now) {
deadline = selectionCooldownDeadline
ok = true
}
if !ok {
return lastErr
}
return &poolCooldownTerminalError{cause: lastErr, deadline: deadline}
}
func (m *Manager) warnLogAuthUnavailable(ctx context.Context, providers []string, model string, opts cliproxyexecutor.Options, tried map[string]struct{}, err error) {
if m == nil || err == nil || !isAuthUnavailableError(err) {
return

View File

@@ -358,6 +358,87 @@ func (e *authUnavailableError) Headers() http.Header {
return safeRetryAfterHeader(e.retryAfter)
}
// poolCooldownTerminalError wraps a terminal upstream 429 when every eligible
// candidate credential for the requested model is blocked and at least one is
// in a timed cooldown. It keeps the upstream error text, body, status, and
// passthrough headers while marking the error as a model cooldown and exposing
// the pool's earliest recovery deadline via Retry-After.
type poolCooldownTerminalError struct {
cause error
deadline time.Time
}
func (e *poolCooldownTerminalError) Error() string {
if e == nil || e.cause == nil {
return ""
}
return e.cause.Error()
}
func (e *poolCooldownTerminalError) Unwrap() error {
if e == nil {
return nil
}
return e.cause
}
func (e *poolCooldownTerminalError) As(target any) bool {
if e == nil {
return false
}
if t, ok := target.(**Error); ok {
var base *Error
if errors.As(e.cause, &base) && base != nil {
*t = base
return true
}
}
return false
}
func (e *poolCooldownTerminalError) IsModelCooldown() bool {
return e != nil && !e.deadline.IsZero()
}
func (e *poolCooldownTerminalError) StatusCode() int {
if e == nil {
return 0
}
return statusCodeFromError(e.cause)
}
func (e *poolCooldownTerminalError) RetryAfter() *time.Duration {
if e == nil || e.deadline.IsZero() {
return nil
}
remaining := time.Until(e.deadline)
if remaining <= 0 {
return nil
}
return &remaining
}
func (e *poolCooldownTerminalError) SafeResponseHeaders() http.Header {
if e == nil || e.deadline.IsZero() {
return nil
}
return safeRetryAfterHeader(time.Until(e.deadline))
}
func (e *poolCooldownTerminalError) Headers() http.Header {
if e == nil || e.cause == nil {
return nil
}
for cur := e.cause; cur != nil; cur = errors.Unwrap(cur) {
if carrier, ok := cur.(interface{ Headers() http.Header }); ok && carrier != nil {
if hdr := carrier.Headers(); len(hdr) > 0 {
return hdr
}
}
}
return nil
}
// trustedRetryAfterError annotates a terminal upstream rejection whose
// locally recorded cooldown deadline governs the client-visible Retry-After.
// The wrapped error keeps its own status, body, identity and retry hint; the
@@ -507,6 +588,10 @@ func HasTrustedRetryDeadline(err error) bool {
if SafeResponseHeaders(err).Get("Retry-After") != "" {
return true
}
var poolCooldown *poolCooldownTerminalError
if errors.As(err, &poolCooldown) && poolCooldown != nil && !poolCooldown.deadline.IsZero() {
return true
}
var trusted *trustedRetryAfterError
return errors.As(err, &trusted) && trusted != nil && !trusted.deadline.IsZero()
}
@@ -518,6 +603,12 @@ func trustedRetryAfterFromError(err error) *time.Duration {
if err == nil {
return nil
}
var poolCooldown *poolCooldownTerminalError
if errors.As(err, &poolCooldown) && poolCooldown != nil {
if remaining := poolCooldown.RetryAfter(); remaining != nil && *remaining > 0 {
return remaining
}
}
var trusted *trustedRetryAfterError
if errors.As(err, &trusted) && trusted != nil {
if remaining := trusted.RetryAfter(); remaining != nil && *remaining > 0 {

View File

@@ -292,6 +292,10 @@ func SafeResponseHeaders(err error) http.Header {
}
return nil
}
var poolCooldown *poolCooldownTerminalError
if errors.As(err, &poolCooldown) && poolCooldown != nil {
return poolCooldown.SafeResponseHeaders()
}
var trusted *trustedRetryAfterError
if errors.As(err, &trusted) && trusted != nil {
return trusted.SafeResponseHeaders()