mirror of
https://github.com/router-for-me/CLIProxyAPI.git
synced 2026-11-18 15:48:39 +08:00
fix(auth): carry the pool cooldown reset on the terminal 429
- Wrap the final 429 in a poolCooldownTerminalError at the Execute/ExecuteCount/ExecuteStream boundaries when every candidate credential for the model is blocked and at least one is in a timed cooldown, preserving the upstream error body and status while marking IsModelCooldown so the Claude handler appends the reset time. - Compute the Retry-After from the earliest auto-recovery across all blocked candidates (including non-cooldown timed blocks) so the client sees the pool's real recovery instead of the last attempt's hint. - Add regression coverage for both cooldown variants across the three execution paths, the available-candidate guard, mixed-block earliest deadlines, the sub-floor hint clamp, and the Claude handler end-to-end. Closes: #7521
This commit is contained in:
174
sdk/api/handlers/claude/code_handlers_pool_cooldown_test.go
Normal file
174
sdk/api/handlers/claude/code_handlers_pool_cooldown_test.go
Normal file
@@ -0,0 +1,174 @@
|
||||
package claude
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/MoeForge/moego/ojson"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
|
||||
"github.com/router-for-me/CLIProxyAPI/v8/sdk/api/handlers"
|
||||
coreauth "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/auth"
|
||||
coreexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
|
||||
sdkconfig "github.com/router-for-me/CLIProxyAPI/v8/sdk/config"
|
||||
)
|
||||
|
||||
type issue7521UpstreamError struct {
|
||||
status int
|
||||
body string
|
||||
retryAfter time.Duration
|
||||
headers http.Header
|
||||
}
|
||||
|
||||
func (e *issue7521UpstreamError) Error() string { return e.body }
|
||||
func (e *issue7521UpstreamError) StatusCode() int { return e.status }
|
||||
func (e *issue7521UpstreamError) RetryAfter() *time.Duration {
|
||||
d := e.retryAfter
|
||||
return &d
|
||||
}
|
||||
func (e *issue7521UpstreamError) Headers() http.Header {
|
||||
if e.headers == nil {
|
||||
return nil
|
||||
}
|
||||
return e.headers.Clone()
|
||||
}
|
||||
|
||||
type issue7521ClaudeExecutor struct {
|
||||
executeFn func(ctx context.Context, auth *coreauth.Auth, req coreexecutor.Request, opts coreexecutor.Options) (coreexecutor.Response, error)
|
||||
streamFn func(ctx context.Context, auth *coreauth.Auth, req coreexecutor.Request, opts coreexecutor.Options) (*coreexecutor.StreamResult, error)
|
||||
countFn func(ctx context.Context, auth *coreauth.Auth, req coreexecutor.Request, opts coreexecutor.Options) (coreexecutor.Response, error)
|
||||
}
|
||||
|
||||
func (e *issue7521ClaudeExecutor) Identifier() string { return "claude" }
|
||||
|
||||
func (e *issue7521ClaudeExecutor) Execute(ctx context.Context, auth *coreauth.Auth, req coreexecutor.Request, opts coreexecutor.Options) (coreexecutor.Response, error) {
|
||||
coreexecutor.MarkUpstreamAttempt(ctx)
|
||||
return e.executeFn(ctx, auth, req, opts)
|
||||
}
|
||||
|
||||
func (e *issue7521ClaudeExecutor) ExecuteStream(ctx context.Context, auth *coreauth.Auth, req coreexecutor.Request, opts coreexecutor.Options) (*coreexecutor.StreamResult, error) {
|
||||
coreexecutor.MarkUpstreamAttempt(ctx)
|
||||
return e.streamFn(ctx, auth, req, opts)
|
||||
}
|
||||
|
||||
func (e *issue7521ClaudeExecutor) CountTokens(ctx context.Context, auth *coreauth.Auth, req coreexecutor.Request, opts coreexecutor.Options) (coreexecutor.Response, error) {
|
||||
coreexecutor.MarkUpstreamAttempt(ctx)
|
||||
return e.countFn(ctx, auth, req, opts)
|
||||
}
|
||||
|
||||
func (e *issue7521ClaudeExecutor) Refresh(_ context.Context, auth *coreauth.Auth) (*coreauth.Auth, error) {
|
||||
return auth, nil
|
||||
}
|
||||
|
||||
func (e *issue7521ClaudeExecutor) HttpRequest(_ context.Context, _ *coreauth.Auth, _ *http.Request) (*http.Response, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestClaudeLastCredential429AppendsPoolResetTime(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
const model = "issue-7521-claude-route-model"
|
||||
shortHint := 95 * time.Minute
|
||||
longHint := 3 * time.Hour
|
||||
|
||||
makeErr := func(authID string) error {
|
||||
if strings.HasSuffix(authID, "-a") {
|
||||
return &issue7521UpstreamError{
|
||||
status: http.StatusTooManyRequests,
|
||||
body: `{"type":"error","error":{"type":"rate_limit_error","message":"First account rate limited."}}`,
|
||||
retryAfter: shortHint,
|
||||
headers: http.Header{"Retry-After": {"5700"}},
|
||||
}
|
||||
}
|
||||
return &issue7521UpstreamError{
|
||||
status: http.StatusTooManyRequests,
|
||||
body: `{"type":"error","error":{"type":"rate_limit_error","message":"This request would exceed your account's monthly spend limit. Please try again later."}}`,
|
||||
retryAfter: longHint,
|
||||
headers: http.Header{"Retry-After": {"10800"}, "X-Upstream-Trace": {"trace-b"}},
|
||||
}
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
path string
|
||||
streaming bool
|
||||
}{
|
||||
{name: "messages_non_stream", path: "/v1/messages", streaming: false},
|
||||
{name: "messages_stream", path: "/v1/messages", streaming: true},
|
||||
{name: "count_tokens", path: "/v1/messages/count_tokens", streaming: false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
manager := coreauth.NewManager(nil, nil, nil)
|
||||
manager.RegisterExecutor(&issue7521ClaudeExecutor{
|
||||
executeFn: func(_ context.Context, auth *coreauth.Auth, _ coreexecutor.Request, _ coreexecutor.Options) (coreexecutor.Response, error) {
|
||||
return coreexecutor.Response{}, makeErr(auth.ID)
|
||||
},
|
||||
streamFn: func(_ context.Context, auth *coreauth.Auth, _ coreexecutor.Request, _ coreexecutor.Options) (*coreexecutor.StreamResult, error) {
|
||||
chunks := make(chan coreexecutor.StreamChunk, 1)
|
||||
chunks <- coreexecutor.StreamChunk{Err: makeErr(auth.ID)}
|
||||
close(chunks)
|
||||
return &coreexecutor.StreamResult{Chunks: chunks}, nil
|
||||
},
|
||||
countFn: func(_ context.Context, auth *coreauth.Auth, _ coreexecutor.Request, _ coreexecutor.Options) (coreexecutor.Response, error) {
|
||||
return coreexecutor.Response{}, makeErr(auth.ID)
|
||||
},
|
||||
})
|
||||
|
||||
reg := registry.GetGlobalRegistry()
|
||||
for _, suffix := range []string{"-a", "-b"} {
|
||||
authID := "issue-7521-route-" + tc.name + suffix
|
||||
auth := &coreauth.Auth{
|
||||
ID: authID,
|
||||
Provider: "claude",
|
||||
Status: coreauth.StatusActive,
|
||||
Attributes: map[string]string{"api_key": "key" + suffix},
|
||||
}
|
||||
if _, errReg := manager.Register(context.Background(), auth); errReg != nil {
|
||||
t.Fatalf("register auth: %v", errReg)
|
||||
}
|
||||
reg.RegisterClient(authID, "claude", []*registry.ModelInfo{{ID: model}})
|
||||
t.Cleanup(func() {
|
||||
reg.UnregisterClient(authID)
|
||||
})
|
||||
}
|
||||
|
||||
handler := NewClaudeCodeAPIHandler(handlers.NewBaseAPIHandlers(&sdkconfig.SDKConfig{PassthroughHeaders: true}, manager))
|
||||
router := gin.New()
|
||||
router.POST("/v1/messages", handler.ClaudeMessages)
|
||||
router.POST("/v1/messages/count_tokens", handler.ClaudeCountTokens)
|
||||
|
||||
body := `{"model":"` + model + `","messages":[{"role":"user","content":"hi"}],"stream":` + strconv.FormatBool(tc.streaming) + `}`
|
||||
req := httptest.NewRequest(http.MethodPost, tc.path, strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusTooManyRequests {
|
||||
t.Fatalf("status = %d, want 429; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
seconds, errParse := strconv.ParseInt(rec.Header().Get("Retry-After"), 10, 64)
|
||||
if errParse != nil || seconds <= 0 {
|
||||
t.Fatalf("Retry-After = %q, want positive integer seconds", rec.Header().Get("Retry-After"))
|
||||
}
|
||||
if seconds > int64((shortHint + 10*time.Second).Seconds()) {
|
||||
t.Fatalf("Retry-After = %d, want pool deadline near %d rather than last attempt 10800", seconds, int64(shortHint.Seconds()))
|
||||
}
|
||||
if gotTrace := rec.Header().Get("X-Upstream-Trace"); gotTrace != "trace-b" {
|
||||
t.Fatalf("X-Upstream-Trace = %q, want trace-b preserved under passthrough-headers", gotTrace)
|
||||
}
|
||||
if gotType := ojson.GetBytesNoCopy(rec.Body.Bytes(), "error.type").String(); gotType != "rate_limit_error" {
|
||||
t.Fatalf("error.type = %q, want rate_limit_error", gotType)
|
||||
}
|
||||
wantDelay := (time.Duration(seconds) * time.Second).String()
|
||||
wantMessage := "This request would exceed your account's monthly spend limit. Please try again later. (reset after " + wantDelay + ")"
|
||||
if gotMsg := ojson.GetBytesNoCopy(rec.Body.Bytes(), "error.message").String(); gotMsg != wantMessage {
|
||||
t.Fatalf("error.message = %q, want %q", gotMsg, wantMessage)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3455,7 +3455,8 @@ func retryAfterFromError(err error) *time.Duration {
|
||||
// An expired trusted wrapper reports a negative remaining time; natural
|
||||
// expiry is not an invalid provider hint, so fall through to the wrapped
|
||||
// error's own hint instead of surfacing the negative value.
|
||||
if _, isTrusted := rap.(*trustedRetryAfterError); isTrusted {
|
||||
switch rap.(type) {
|
||||
case *poolCooldownTerminalError, *trustedRetryAfterError:
|
||||
if inner := errors.Unwrap(err); inner != nil {
|
||||
return retryAfterFromError(inner)
|
||||
}
|
||||
|
||||
@@ -224,6 +224,7 @@ func (m *Manager) Execute(ctx context.Context, providers []string, req cliproxye
|
||||
|
||||
var lastErr error
|
||||
var preferredUpstreamErr error
|
||||
var selectionCooldownDeadline time.Time
|
||||
var terminalRoundRejectedByScheduler bool
|
||||
retryModel := authSelectionModelFromOptions(opts, req.Model)
|
||||
for attempt := 0; ; attempt++ {
|
||||
@@ -244,6 +245,7 @@ func (m *Manager) Execute(ctx context.Context, providers []string, req cliproxye
|
||||
break
|
||||
}
|
||||
terminalRoundRejectedByScheduler = hasRoundEndedBySchedulerReject(errExec)
|
||||
observeSelectionCooldownDeadline(&selectionCooldownDeadline, errExec, time.Now())
|
||||
if hasUpstreamExecutionAttempt(errExec) {
|
||||
preferredUpstreamErr = errExec
|
||||
}
|
||||
@@ -272,6 +274,7 @@ func (m *Manager) Execute(ctx context.Context, providers []string, req cliproxye
|
||||
return resp, nil
|
||||
}
|
||||
}
|
||||
lastErr = m.annotatePoolCooldownTerminalError(ctx, normalized, retryModel, opts, lastErr, selectionCooldownDeadline)
|
||||
return cliproxyexecutor.Response{}, lastErr
|
||||
}
|
||||
return cliproxyexecutor.Response{}, &Error{Code: "auth_not_found", Message: "no auth available"}
|
||||
@@ -299,6 +302,7 @@ func (m *Manager) ExecuteCount(ctx context.Context, providers []string, req clip
|
||||
|
||||
var lastErr error
|
||||
var preferredUpstreamErr error
|
||||
var selectionCooldownDeadline time.Time
|
||||
retryModel := authSelectionModelFromOptions(opts, req.Model)
|
||||
for attempt := 0; ; attempt++ {
|
||||
roundAttempted := make(map[string]struct{})
|
||||
@@ -316,6 +320,7 @@ func (m *Manager) ExecuteCount(ctx context.Context, providers []string, req clip
|
||||
}
|
||||
break
|
||||
}
|
||||
observeSelectionCooldownDeadline(&selectionCooldownDeadline, errExec, time.Now())
|
||||
if hasUpstreamExecutionAttempt(errExec) {
|
||||
preferredUpstreamErr = errExec
|
||||
}
|
||||
@@ -337,6 +342,7 @@ func (m *Manager) ExecuteCount(ctx context.Context, providers []string, req clip
|
||||
lastErr = preferredExecutionAttemptError(lastErr, preferredUpstreamErr)
|
||||
lastErr = unwrapExecutionBoundaryError(lastErr)
|
||||
lastErr = m.normalizeSessionAffinityNeverTerminalError(lastErr, normalized, retryModel, opts)
|
||||
lastErr = m.annotatePoolCooldownTerminalError(ctx, normalized, retryModel, opts, lastErr, selectionCooldownDeadline)
|
||||
return cliproxyexecutor.Response{}, lastErr
|
||||
}
|
||||
return cliproxyexecutor.Response{}, &Error{Code: "auth_not_found", Message: "no auth available"}
|
||||
@@ -375,6 +381,7 @@ func (m *Manager) ExecuteStream(ctx context.Context, providers []string, req cli
|
||||
|
||||
var lastErr error
|
||||
var preferredUpstreamErr error
|
||||
var selectionCooldownDeadline time.Time
|
||||
var terminalRoundRejectedByScheduler bool
|
||||
homeRetryLimit := -1
|
||||
retryModel := authSelectionModelFromOptions(opts, req.Model)
|
||||
@@ -388,6 +395,9 @@ func (m *Manager) ExecuteStream(ctx context.Context, providers []string, req cli
|
||||
if errStream == nil {
|
||||
return result, nil
|
||||
}
|
||||
if !m.HomeEnabled() {
|
||||
observeSelectionCooldownDeadline(&selectionCooldownDeadline, errStream, time.Now())
|
||||
}
|
||||
if hasUpstreamExecutionAttempt(errStream) {
|
||||
preferredUpstreamErr = errStream
|
||||
}
|
||||
@@ -446,6 +456,9 @@ func (m *Manager) ExecuteStream(ctx context.Context, providers []string, req cli
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
if !m.HomeEnabled() {
|
||||
lastErr = m.annotatePoolCooldownTerminalError(ctx, normalized, retryModel, opts, lastErr, selectionCooldownDeadline)
|
||||
}
|
||||
var bootstrapErr *streamBootstrapError
|
||||
if errors.As(lastErr, &bootstrapErr) && bootstrapErr != nil {
|
||||
return streamErrorResult(bootstrapErr.Headers(), lastErr), nil
|
||||
|
||||
459
sdk/cliproxy/auth/conductor_pool_cooldown_test.go
Normal file
459
sdk/cliproxy/auth/conductor_pool_cooldown_test.go
Normal file
@@ -0,0 +1,459 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
|
||||
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
|
||||
)
|
||||
|
||||
const issue7521PoolModel = "claude-sonnet-4-6-issue-7521"
|
||||
|
||||
func newIssue7521Manager(
|
||||
t *testing.T,
|
||||
selector Selector,
|
||||
exec ProviderExecutor,
|
||||
auths ...*Auth,
|
||||
) *Manager {
|
||||
t.Helper()
|
||||
withQuotaCooldownEnabled(t)
|
||||
if selector == nil {
|
||||
selector = &RoundRobinSelector{}
|
||||
}
|
||||
manager := NewManager(nil, selector, nil)
|
||||
manager.SetRetryConfig(2, 30*time.Second, 0)
|
||||
manager.RegisterExecutor(exec)
|
||||
reg := registry.GetGlobalRegistry()
|
||||
for _, auth := range auths {
|
||||
authID := auth.ID
|
||||
reg.RegisterClient(authID, auth.Provider, []*registry.ModelInfo{{ID: issue7521PoolModel}})
|
||||
t.Cleanup(func() {
|
||||
reg.UnregisterClient(authID)
|
||||
})
|
||||
if _, errRegister := manager.Register(context.Background(), auth); errRegister != nil {
|
||||
t.Fatalf("register auth %s: %v", authID, errRegister)
|
||||
}
|
||||
}
|
||||
return manager
|
||||
}
|
||||
|
||||
func newIssue7521Auth(id string) *Auth {
|
||||
return &Auth{
|
||||
ID: id,
|
||||
Provider: "claude",
|
||||
Status: StatusActive,
|
||||
Attributes: map[string]string{"api_key": "key-" + id},
|
||||
}
|
||||
}
|
||||
|
||||
func newIssue7521Upstream429(msg string, retryAfter time.Duration) error {
|
||||
hint := retryAfter
|
||||
return customStatusError{
|
||||
code: http.StatusTooManyRequests,
|
||||
msg: msg,
|
||||
retryAfter: &hint,
|
||||
}
|
||||
}
|
||||
|
||||
func assertIssue7521PoolCooldownError(t *testing.T, err error, wantMessage string, wantDeadline time.Duration) {
|
||||
t.Helper()
|
||||
if err == nil {
|
||||
t.Fatal("expected terminal 429 error, got nil")
|
||||
}
|
||||
if got := statusCodeFromError(err); got != http.StatusTooManyRequests {
|
||||
t.Fatalf("status = %d, want 429", got)
|
||||
}
|
||||
if !strings.Contains(err.Error(), wantMessage) {
|
||||
t.Fatalf("error = %q, want upstream message %q preserved", err.Error(), wantMessage)
|
||||
}
|
||||
var marker interface{ IsModelCooldown() bool }
|
||||
if !errors.As(err, &marker) || marker == nil || !marker.IsModelCooldown() {
|
||||
t.Fatalf("error %T (%v) must report IsModelCooldown() == true", err, err)
|
||||
}
|
||||
if !HasTrustedRetryDeadline(err) {
|
||||
t.Fatal("HasTrustedRetryDeadline(err) = false, want true")
|
||||
}
|
||||
retryAfter := SafeResponseHeaders(err).Get("Retry-After")
|
||||
seconds, errParse := strconv.ParseInt(retryAfter, 10, 64)
|
||||
if errParse != nil || seconds <= 0 {
|
||||
t.Fatalf("Retry-After = %q, want positive integer seconds", retryAfter)
|
||||
}
|
||||
gotDuration := time.Duration(seconds) * time.Second
|
||||
if gotDuration > wantDeadline+10*time.Second || gotDuration < wantDeadline-10*time.Second {
|
||||
t.Fatalf("Retry-After = %ds (%s), want near pool recovery %s", seconds, gotDuration, wantDeadline)
|
||||
}
|
||||
}
|
||||
|
||||
// Variant 2: fresh start where every candidate credential is tried in the same
|
||||
// round and returns 429; no modelCooldownError is built during selection, yet
|
||||
// the terminal 429 must report IsModelCooldown() and carry the earliest pool reset.
|
||||
func TestExecuteLastCredential429CarriesEarliestPoolReset(t *testing.T) {
|
||||
first := newIssue7521Auth("7521-exec-a")
|
||||
second := newIssue7521Auth("7521-exec-b")
|
||||
shortHint := 90 * time.Minute
|
||||
longHint := 3 * time.Hour
|
||||
|
||||
var attempts []string
|
||||
exec := &customStreamMockExecutor{
|
||||
identifier: "claude",
|
||||
mockCustomErrorExecutor: mockCustomErrorExecutor{
|
||||
identifier: "claude",
|
||||
executeFn: func(ctx context.Context, auth *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
|
||||
cliproxyexecutor.MarkUpstreamAttempt(ctx)
|
||||
attempts = append(attempts, auth.ID)
|
||||
if auth.ID == first.ID {
|
||||
return cliproxyexecutor.Response{}, newIssue7521Upstream429("first credential rate limited", shortHint)
|
||||
}
|
||||
return cliproxyexecutor.Response{}, newIssue7521Upstream429("monthly spend limit reached", longHint)
|
||||
},
|
||||
},
|
||||
}
|
||||
manager := newIssue7521Manager(t, nil, exec, first, second)
|
||||
|
||||
_, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{})
|
||||
assertIssue7521PoolCooldownError(t, errExec, "monthly spend limit reached", shortHint)
|
||||
if len(attempts) != 2 {
|
||||
t.Fatalf("attempts = %v, want 2 credentials tried", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSingleCredentialSubFloorHintReportsActualPoolRecovery(t *testing.T) {
|
||||
single := newIssue7521Auth("7521-subfloor-a")
|
||||
hint := 7 * time.Second
|
||||
exec := &customStreamMockExecutor{
|
||||
identifier: "claude",
|
||||
mockCustomErrorExecutor: mockCustomErrorExecutor{
|
||||
identifier: "claude",
|
||||
executeFn: func(ctx context.Context, _ *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
|
||||
cliproxyexecutor.MarkUpstreamAttempt(ctx)
|
||||
return cliproxyexecutor.Response{}, newIssue7521Upstream429("try later", hint)
|
||||
},
|
||||
},
|
||||
}
|
||||
manager := newIssue7521Manager(t, nil, exec, single)
|
||||
manager.SetRetryConfig(0, 0, 0)
|
||||
|
||||
_, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{})
|
||||
if errExec == nil {
|
||||
t.Fatal("expected terminal 429 error")
|
||||
}
|
||||
// MarkResult clamps the 7s hint up to minQuotaCooldownFloor (10s), so the
|
||||
// pool's actual recovery is 10s out. The wrap must advertise that real
|
||||
// deadline, not the shorter per-attempt hint.
|
||||
if got := SafeResponseHeaders(errExec).Get("Retry-After"); got != "10" {
|
||||
t.Fatalf("Retry-After = %q, want 10 (minQuotaCooldownFloor)", got)
|
||||
}
|
||||
var marker interface{ IsModelCooldown() bool }
|
||||
if !errors.As(errExec, &marker) || marker == nil || !marker.IsModelCooldown() {
|
||||
t.Fatalf("error %T must report IsModelCooldown() == true", errExec)
|
||||
}
|
||||
}
|
||||
|
||||
// Variant 1: one credential is already cooling down; the remaining credential
|
||||
// returns 429 and the next pick fails with modelCooldownError, which is
|
||||
// replaced by preferredExecutionAttemptError. The terminal 429 must still keep
|
||||
// the upstream text and carry the earliest pool cooldown reset.
|
||||
func TestExecuteLastUsableCredential429KeepsExistingCooldownReset(t *testing.T) {
|
||||
first := newIssue7521Auth("7521-var1-a")
|
||||
second := newIssue7521Auth("7521-var1-b")
|
||||
existingCooldown := 75 * time.Minute
|
||||
freshHint := 3 * time.Hour
|
||||
|
||||
exec := &customStreamMockExecutor{
|
||||
identifier: "claude",
|
||||
mockCustomErrorExecutor: mockCustomErrorExecutor{
|
||||
identifier: "claude",
|
||||
executeFn: func(ctx context.Context, _ *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
|
||||
cliproxyexecutor.MarkUpstreamAttempt(ctx)
|
||||
return cliproxyexecutor.Response{}, newIssue7521Upstream429("second credential rate limited", freshHint)
|
||||
},
|
||||
},
|
||||
}
|
||||
manager := newIssue7521Manager(t, nil, exec, first, second)
|
||||
manager.MarkResult(context.Background(), Result{
|
||||
AuthID: first.ID,
|
||||
Provider: "claude",
|
||||
Model: issue7521PoolModel,
|
||||
Success: false,
|
||||
RetryAfter: &existingCooldown,
|
||||
CredentialScope: true,
|
||||
Error: &Error{HTTPStatus: http.StatusTooManyRequests, Message: "already cooling"},
|
||||
})
|
||||
|
||||
_, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{})
|
||||
assertIssue7521PoolCooldownError(t, errExec, "second credential rate limited", existingCooldown)
|
||||
}
|
||||
|
||||
func TestExecuteCountLastCredential429CarriesEarliestPoolReset(t *testing.T) {
|
||||
first := newIssue7521Auth("7521-count-a")
|
||||
second := newIssue7521Auth("7521-count-b")
|
||||
shortHint := 85 * time.Minute
|
||||
longHint := 4 * time.Hour
|
||||
|
||||
exec := &customStreamMockExecutor{
|
||||
identifier: "claude",
|
||||
mockCustomErrorExecutor: mockCustomErrorExecutor{
|
||||
identifier: "claude",
|
||||
countFn: func(ctx context.Context, auth *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
|
||||
cliproxyexecutor.MarkUpstreamAttempt(ctx)
|
||||
if auth.ID == first.ID {
|
||||
return cliproxyexecutor.Response{}, newIssue7521Upstream429("count first limited", shortHint)
|
||||
}
|
||||
return cliproxyexecutor.Response{}, newIssue7521Upstream429("count second limited", longHint)
|
||||
},
|
||||
},
|
||||
}
|
||||
manager := newIssue7521Manager(t, nil, exec, first, second)
|
||||
|
||||
_, errCount := manager.ExecuteCount(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{})
|
||||
assertIssue7521PoolCooldownError(t, errCount, "count second limited", shortHint)
|
||||
}
|
||||
|
||||
func TestExecuteStreamLastCredential429CarriesEarliestPoolReset(t *testing.T) {
|
||||
shortHint := 80 * time.Minute
|
||||
longHint := 3 * time.Hour
|
||||
|
||||
t.Run("direct stream error", func(t *testing.T) {
|
||||
first := newIssue7521Auth("7521-stream-direct-a")
|
||||
second := newIssue7521Auth("7521-stream-direct-b")
|
||||
exec := &customStreamMockExecutor{
|
||||
identifier: "claude",
|
||||
streamFn: func(ctx context.Context, auth *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) {
|
||||
cliproxyexecutor.MarkUpstreamAttempt(ctx)
|
||||
if auth.ID == first.ID {
|
||||
return nil, newIssue7521Upstream429("stream first limited", shortHint)
|
||||
}
|
||||
return nil, newIssue7521Upstream429("stream second limited", longHint)
|
||||
},
|
||||
}
|
||||
manager := newIssue7521Manager(t, nil, exec, first, second)
|
||||
|
||||
_, errStream := manager.ExecuteStream(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{Stream: true})
|
||||
assertIssue7521PoolCooldownError(t, errStream, "stream second limited", shortHint)
|
||||
})
|
||||
|
||||
t.Run("bootstrap chunk error", func(t *testing.T) {
|
||||
first := newIssue7521Auth("7521-stream-boot-a")
|
||||
second := newIssue7521Auth("7521-stream-boot-b")
|
||||
exec := &customStreamMockExecutor{
|
||||
identifier: "claude",
|
||||
streamFn: func(ctx context.Context, auth *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) {
|
||||
cliproxyexecutor.MarkUpstreamAttempt(ctx)
|
||||
chunks := make(chan cliproxyexecutor.StreamChunk, 1)
|
||||
if auth.ID == first.ID {
|
||||
chunks <- cliproxyexecutor.StreamChunk{Err: newIssue7521Upstream429("bootstrap first limited", shortHint)}
|
||||
} else {
|
||||
chunks <- cliproxyexecutor.StreamChunk{Err: newIssue7521Upstream429("bootstrap second limited", longHint)}
|
||||
}
|
||||
close(chunks)
|
||||
return &cliproxyexecutor.StreamResult{Chunks: chunks}, nil
|
||||
},
|
||||
}
|
||||
manager := newIssue7521Manager(t, nil, exec, first, second)
|
||||
|
||||
streamRes, errStream := manager.ExecuteStream(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{Stream: true})
|
||||
if errStream != nil {
|
||||
t.Fatalf("expected streamErrorResult, got direct err: %v", errStream)
|
||||
}
|
||||
chunk, ok := <-streamRes.Chunks
|
||||
if !ok {
|
||||
t.Fatal("expected error chunk on bootstrap failure")
|
||||
}
|
||||
assertIssue7521PoolCooldownError(t, chunk.Err, "bootstrap second limited", shortHint)
|
||||
})
|
||||
}
|
||||
|
||||
func TestExecute429LeavesOrdinary429UnchangedWhenCandidateAvailable(t *testing.T) {
|
||||
first := newIssue7521Auth("7521-avail-a")
|
||||
second := newIssue7521Auth("7521-avail-b")
|
||||
hint := time.Hour
|
||||
|
||||
exec := &customStreamMockExecutor{
|
||||
identifier: "claude",
|
||||
mockCustomErrorExecutor: mockCustomErrorExecutor{
|
||||
identifier: "claude",
|
||||
executeFn: func(ctx context.Context, _ *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
|
||||
cliproxyexecutor.MarkUpstreamAttempt(ctx)
|
||||
return cliproxyexecutor.Response{}, newIssue7521Upstream429("single attempt 429", hint)
|
||||
},
|
||||
},
|
||||
}
|
||||
manager := newIssue7521Manager(t, nil, exec, first, second)
|
||||
// Stop after 1 credential attempt with 0 retries so the second credential remains unblocked.
|
||||
manager.SetRetryConfig(0, 0, 1)
|
||||
|
||||
_, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{})
|
||||
if errExec == nil {
|
||||
t.Fatal("expected 429 error")
|
||||
}
|
||||
if got := statusCodeFromError(errExec); got != http.StatusTooManyRequests {
|
||||
t.Fatalf("status = %d, want 429", got)
|
||||
}
|
||||
var marker interface{ IsModelCooldown() bool }
|
||||
if errors.As(errExec, &marker) && marker != nil && marker.IsModelCooldown() {
|
||||
t.Fatalf("ordinary 429 must not report IsModelCooldown() when another candidate is still available: %T (%v)", errExec, errExec)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteLastCredential429IgnoresZeroWeightCandidateUnderWeightedSelector(t *testing.T) {
|
||||
active := newIssue7521Auth("7521-weight-active")
|
||||
drained := newIssue7521Auth("7521-weight-zero")
|
||||
drained.Attributes["weight"] = "0"
|
||||
hint := 70 * time.Minute
|
||||
|
||||
exec := &customStreamMockExecutor{
|
||||
identifier: "claude",
|
||||
mockCustomErrorExecutor: mockCustomErrorExecutor{
|
||||
identifier: "claude",
|
||||
executeFn: func(ctx context.Context, _ *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
|
||||
cliproxyexecutor.MarkUpstreamAttempt(ctx)
|
||||
return cliproxyexecutor.Response{}, newIssue7521Upstream429("active credential exhausted", hint)
|
||||
},
|
||||
},
|
||||
}
|
||||
selector := NewSessionAffinitySelector(&WeightedRoundRobinSelector{})
|
||||
manager := newIssue7521Manager(t, selector, exec, active, drained)
|
||||
|
||||
_, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{})
|
||||
assertIssue7521PoolCooldownError(t, errExec, "active credential exhausted", hint)
|
||||
}
|
||||
|
||||
func TestAnnotatePoolCooldownSelectionFallbackAndUnblockedGuard(t *testing.T) {
|
||||
now := time.Date(2026, 11, 14, 12, 0, 0, 0, time.UTC)
|
||||
fallbackDeadline := now.Add(42 * time.Minute)
|
||||
lastAttemptHint := 3 * time.Hour
|
||||
baseErr := withTrustedRetryAfter(newIssue7521Upstream429("upstream rate limited", lastAttemptHint), &lastAttemptHint)
|
||||
|
||||
t.Run("falls back to selection cooldown when no candidate unblocked", func(t *testing.T) {
|
||||
manager := NewManager(nil, nil, nil)
|
||||
wrapped := manager.annotatePoolCooldownTerminalErrorAt(
|
||||
context.Background(),
|
||||
[]string{"claude"},
|
||||
issue7521PoolModel,
|
||||
cliproxyexecutor.Options{},
|
||||
baseErr,
|
||||
fallbackDeadline,
|
||||
now,
|
||||
)
|
||||
poolErr, ok := wrapped.(*poolCooldownTerminalError)
|
||||
if !ok || poolErr == nil {
|
||||
t.Fatalf("wrapped = %T, want *poolCooldownTerminalError", wrapped)
|
||||
}
|
||||
if !poolErr.deadline.Equal(fallbackDeadline) {
|
||||
t.Fatalf("deadline = %v, want %v", poolErr.deadline, fallbackDeadline)
|
||||
}
|
||||
if !poolErr.IsModelCooldown() {
|
||||
t.Fatal("IsModelCooldown() = false, want true")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("does not fall back when an eligible candidate is unblocked", func(t *testing.T) {
|
||||
avail := newIssue7521Auth("7521-fallback-avail")
|
||||
exec := &customStreamMockExecutor{identifier: "claude"}
|
||||
manager := newIssue7521Manager(t, nil, exec, avail)
|
||||
|
||||
got := manager.annotatePoolCooldownTerminalErrorAt(
|
||||
context.Background(),
|
||||
[]string{"claude"},
|
||||
issue7521PoolModel,
|
||||
cliproxyexecutor.Options{},
|
||||
baseErr,
|
||||
fallbackDeadline,
|
||||
now,
|
||||
)
|
||||
var marker interface{ IsModelCooldown() bool }
|
||||
if errors.As(got, &marker) && marker != nil && marker.IsModelCooldown() {
|
||||
t.Fatalf("must not wrap as model cooldown when a candidate is unblocked: %T", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("expired pool deadline suppresses raw header and falls back for retryAfterFromError", func(t *testing.T) {
|
||||
cause := headerCarryingError{headers: http.Header{"X-Trace": {"1"}, "Retry-After": {"999"}}}
|
||||
expired := &poolCooldownTerminalError{
|
||||
cause: cause,
|
||||
deadline: time.Now().Add(-time.Second),
|
||||
}
|
||||
if !HasTrustedRetryDeadline(expired) {
|
||||
t.Fatal("HasTrustedRetryDeadline(expired) = false, want true")
|
||||
}
|
||||
if got := SafeResponseHeaders(expired).Get("Retry-After"); got != "" {
|
||||
t.Fatalf("SafeResponseHeaders(expired) Retry-After = %q, want empty", got)
|
||||
}
|
||||
if got := expired.Headers().Get("X-Trace"); got != "1" {
|
||||
t.Fatalf("Headers() X-Trace = %q, want 1", got)
|
||||
}
|
||||
agg := 2 * time.Second
|
||||
homeTerminal := markHomeRetryRoundExhausted(&poolCooldownTerminalError{
|
||||
cause: cause,
|
||||
deadline: time.Now().Add(time.Hour),
|
||||
}, &agg, false)
|
||||
if got := SafeResponseHeaders(homeTerminal).Get("Retry-After"); got != "2" {
|
||||
t.Fatalf("Home aggregate Retry-After = %q, want 2", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Regression for the review finding: a short transient non-cooldown block on a
|
||||
// sibling credential must pull the pool reset forward instead of being ignored,
|
||||
// so the wrapped terminal 429 never advertises a longer wait than the pool's
|
||||
// earliest auto-recovery.
|
||||
func TestExecutePoolCooldownIncludesTransientNonCooldownBlock(t *testing.T) {
|
||||
longQuotaCooldown := 3 * time.Hour
|
||||
shortTransient := 90 * time.Second
|
||||
|
||||
quotaAuth := newIssue7521Auth("7521-mixed-quota")
|
||||
transientAuth := newIssue7521Auth("7521-mixed-transient")
|
||||
execAuth := newIssue7521Auth("7521-mixed-exec")
|
||||
|
||||
exec := &customStreamMockExecutor{
|
||||
identifier: "claude",
|
||||
mockCustomErrorExecutor: mockCustomErrorExecutor{
|
||||
identifier: "claude",
|
||||
executeFn: func(ctx context.Context, _ *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
|
||||
cliproxyexecutor.MarkUpstreamAttempt(ctx)
|
||||
return cliproxyexecutor.Response{}, newIssue7521Upstream429("last credential rate limited", 2*time.Hour)
|
||||
},
|
||||
},
|
||||
}
|
||||
manager := newIssue7521Manager(t, nil, exec, quotaAuth, transientAuth, execAuth)
|
||||
|
||||
// Credential A: genuine quota cooldown recovering in 3h.
|
||||
manager.MarkResult(context.Background(), Result{
|
||||
AuthID: quotaAuth.ID,
|
||||
Provider: "claude",
|
||||
Model: issue7521PoolModel,
|
||||
Success: false,
|
||||
RetryAfter: &longQuotaCooldown,
|
||||
CredentialScope: true,
|
||||
Error: &Error{HTTPStatus: http.StatusTooManyRequests, Message: "quota exhausted"},
|
||||
})
|
||||
|
||||
// Credential B: transient non-cooldown block recovering in 90s. Recording
|
||||
// a transient_rate_limit result (non-credential scope) marks only the
|
||||
// reporting model unavailable without a quota label, so the scan
|
||||
// classifies it as blockReasonOther rather than blockReasonCooldown.
|
||||
manager.MarkResult(context.Background(), Result{
|
||||
AuthID: transientAuth.ID,
|
||||
Provider: "claude",
|
||||
Model: issue7521PoolModel,
|
||||
Success: false,
|
||||
RetryAfter: &shortTransient,
|
||||
Error: &Error{
|
||||
Code: ErrorCodeTransientRateLimit,
|
||||
HTTPStatus: http.StatusTooManyRequests,
|
||||
Message: "transient capacity rejection",
|
||||
},
|
||||
})
|
||||
|
||||
// Credential C is the only unblocked candidate; it fails and joins the
|
||||
// pool cooldown. The pool's earliest recovery is B's +90s, not A's +3h nor
|
||||
// the attempt's own 2h hint.
|
||||
_, errExec := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: issue7521PoolModel}, cliproxyexecutor.Options{})
|
||||
assertIssue7521PoolCooldownError(t, errExec, "last credential rate limited", shortTransient)
|
||||
}
|
||||
@@ -3012,6 +3012,144 @@ func authCoolingSummary(auth *Auth, model string, next time.Time, now time.Time)
|
||||
return fmt.Sprintf("[%s, reason=%s, remaining=%s]", ident, reason, remaining)
|
||||
}
|
||||
|
||||
func selectorExcludesZeroWeight(selector Selector) bool {
|
||||
if _, weighted := selector.(*WeightedRoundRobinSelector); weighted {
|
||||
return true
|
||||
}
|
||||
if affinity, ok := selector.(*SessionAffinitySelector); ok && affinity != nil {
|
||||
_, weighted := affinity.fallback.(*WeightedRoundRobinSelector)
|
||||
return weighted
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// earliestPoolCooldownForModelAt scans the model's candidate credentials with
|
||||
// the same filters selection uses. It reports:
|
||||
// - earliest: the earliest active timed cooldown recovery instant
|
||||
// - hasUnblocked: true when at least one eligible candidate is still available
|
||||
// - ok: true when no candidate is available and at least one is in a timed cooldown
|
||||
func (m *Manager) earliestPoolCooldownForModelAt(ctx context.Context, providers []string, model string, opts cliproxyexecutor.Options, now time.Time) (earliest time.Time, hasUnblocked bool, ok bool) {
|
||||
if m == nil {
|
||||
return time.Time{}, false, false
|
||||
}
|
||||
excludeZeroWeight := selectorExcludesZeroWeight(m.Selector())
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
eligibility := authSelectionEligibilityForRequest(ctx, opts)
|
||||
pinnedAuthID := pinnedAuthIDFromMetadata(opts.Metadata)
|
||||
providerSet := make(map[string]struct{}, len(providers))
|
||||
for _, p := range providers {
|
||||
if norm := canonicalSchedulingProvider(p); norm != "" && norm != "mixed" {
|
||||
providerSet[norm] = struct{}{}
|
||||
}
|
||||
}
|
||||
registryRef := registry.GetGlobalRegistry()
|
||||
trimmedModel := strings.TrimSpace(model)
|
||||
|
||||
totalCandidates := 0
|
||||
cooling := 0
|
||||
for _, candidate := range m.auths {
|
||||
if candidate == nil || candidate.Disabled {
|
||||
continue
|
||||
}
|
||||
if excludeZeroWeight && authWeight(candidate) <= 0 {
|
||||
continue
|
||||
}
|
||||
rawProvider := executorKeyFromAuth(candidate)
|
||||
providerKey := canonicalSchedulingProvider(rawProvider)
|
||||
if providerKey == "" {
|
||||
continue
|
||||
}
|
||||
if len(providerSet) > 0 {
|
||||
if _, match := providerSet[providerKey]; !match {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if _, hasExec := m.executorLocked(rawProvider); !hasExec {
|
||||
if _, hasNormExec := m.executorLocked(providerKey); !hasNormExec {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if pinnedAuthID != "" && candidate.ID != pinnedAuthID {
|
||||
continue
|
||||
}
|
||||
if !eligibility.allows(candidate) {
|
||||
continue
|
||||
}
|
||||
if trimmedModel != "" && !m.authSupportsRouteModel(registryRef, candidate, model) {
|
||||
continue
|
||||
}
|
||||
totalCandidates++
|
||||
checkModel := m.selectionModelForAuth(candidate, model)
|
||||
blocked, reason, next := isAuthBlockedForModelForRequest(ctx, candidate, checkModel, now, opts.Headers)
|
||||
if !blocked {
|
||||
return time.Time{}, true, false
|
||||
}
|
||||
if reason == blockReasonCooldown {
|
||||
cooling++
|
||||
}
|
||||
// Count every timed auto-recovery toward the pool's earliest reset,
|
||||
// including transient non-cooldown blocks (e.g. a short capacity 429),
|
||||
// matching how selection computes the earliest availability.
|
||||
if reason != blockReasonDisabled && !next.IsZero() && next.After(now) {
|
||||
if earliest.IsZero() || next.Before(earliest) {
|
||||
earliest = next
|
||||
}
|
||||
}
|
||||
}
|
||||
if totalCandidates > 0 && cooling > 0 && !earliest.IsZero() {
|
||||
return earliest, false, true
|
||||
}
|
||||
return time.Time{}, false, false
|
||||
}
|
||||
|
||||
func observeSelectionCooldownDeadline(dst *time.Time, err error, now time.Time) {
|
||||
if dst == nil || err == nil {
|
||||
return
|
||||
}
|
||||
var cooldown *modelCooldownError
|
||||
if !errors.As(err, &cooldown) || cooldown == nil || cooldown.resetIn <= 0 {
|
||||
return
|
||||
}
|
||||
candidate := now.Add(cooldown.resetIn)
|
||||
if dst.IsZero() || candidate.Before(*dst) {
|
||||
*dst = candidate
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) annotatePoolCooldownTerminalError(ctx context.Context, providers []string, model string, opts cliproxyexecutor.Options, lastErr error, selectionCooldownDeadline time.Time) error {
|
||||
return m.annotatePoolCooldownTerminalErrorAt(ctx, providers, model, opts, lastErr, selectionCooldownDeadline, time.Now())
|
||||
}
|
||||
|
||||
func (m *Manager) annotatePoolCooldownTerminalErrorAt(ctx context.Context, providers []string, model string, opts cliproxyexecutor.Options, lastErr error, selectionCooldownDeadline time.Time, now time.Time) error {
|
||||
if m == nil || lastErr == nil {
|
||||
return lastErr
|
||||
}
|
||||
if statusCodeFromError(lastErr) != http.StatusTooManyRequests {
|
||||
return lastErr
|
||||
}
|
||||
var marker interface{ IsModelCooldown() bool }
|
||||
if errors.As(lastErr, &marker) && marker != nil && marker.IsModelCooldown() {
|
||||
return lastErr
|
||||
}
|
||||
var existing *poolCooldownTerminalError
|
||||
if errors.As(lastErr, &existing) && existing != nil {
|
||||
return lastErr
|
||||
}
|
||||
deadline, hasUnblocked, ok := m.earliestPoolCooldownForModelAt(ctx, providers, model, opts, now)
|
||||
if hasUnblocked {
|
||||
return lastErr
|
||||
}
|
||||
if !ok && !selectionCooldownDeadline.IsZero() && selectionCooldownDeadline.After(now) {
|
||||
deadline = selectionCooldownDeadline
|
||||
ok = true
|
||||
}
|
||||
if !ok {
|
||||
return lastErr
|
||||
}
|
||||
return &poolCooldownTerminalError{cause: lastErr, deadline: deadline}
|
||||
}
|
||||
|
||||
func (m *Manager) warnLogAuthUnavailable(ctx context.Context, providers []string, model string, opts cliproxyexecutor.Options, tried map[string]struct{}, err error) {
|
||||
if m == nil || err == nil || !isAuthUnavailableError(err) {
|
||||
return
|
||||
|
||||
@@ -358,6 +358,87 @@ func (e *authUnavailableError) Headers() http.Header {
|
||||
return safeRetryAfterHeader(e.retryAfter)
|
||||
}
|
||||
|
||||
// poolCooldownTerminalError wraps a terminal upstream 429 when every eligible
|
||||
// candidate credential for the requested model is blocked and at least one is
|
||||
// in a timed cooldown. It keeps the upstream error text, body, status, and
|
||||
// passthrough headers while marking the error as a model cooldown and exposing
|
||||
// the pool's earliest recovery deadline via Retry-After.
|
||||
type poolCooldownTerminalError struct {
|
||||
cause error
|
||||
deadline time.Time
|
||||
}
|
||||
|
||||
func (e *poolCooldownTerminalError) Error() string {
|
||||
if e == nil || e.cause == nil {
|
||||
return ""
|
||||
}
|
||||
return e.cause.Error()
|
||||
}
|
||||
|
||||
func (e *poolCooldownTerminalError) Unwrap() error {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return e.cause
|
||||
}
|
||||
|
||||
func (e *poolCooldownTerminalError) As(target any) bool {
|
||||
if e == nil {
|
||||
return false
|
||||
}
|
||||
if t, ok := target.(**Error); ok {
|
||||
var base *Error
|
||||
if errors.As(e.cause, &base) && base != nil {
|
||||
*t = base
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (e *poolCooldownTerminalError) IsModelCooldown() bool {
|
||||
return e != nil && !e.deadline.IsZero()
|
||||
}
|
||||
|
||||
func (e *poolCooldownTerminalError) StatusCode() int {
|
||||
if e == nil {
|
||||
return 0
|
||||
}
|
||||
return statusCodeFromError(e.cause)
|
||||
}
|
||||
|
||||
func (e *poolCooldownTerminalError) RetryAfter() *time.Duration {
|
||||
if e == nil || e.deadline.IsZero() {
|
||||
return nil
|
||||
}
|
||||
remaining := time.Until(e.deadline)
|
||||
if remaining <= 0 {
|
||||
return nil
|
||||
}
|
||||
return &remaining
|
||||
}
|
||||
|
||||
func (e *poolCooldownTerminalError) SafeResponseHeaders() http.Header {
|
||||
if e == nil || e.deadline.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return safeRetryAfterHeader(time.Until(e.deadline))
|
||||
}
|
||||
|
||||
func (e *poolCooldownTerminalError) Headers() http.Header {
|
||||
if e == nil || e.cause == nil {
|
||||
return nil
|
||||
}
|
||||
for cur := e.cause; cur != nil; cur = errors.Unwrap(cur) {
|
||||
if carrier, ok := cur.(interface{ Headers() http.Header }); ok && carrier != nil {
|
||||
if hdr := carrier.Headers(); len(hdr) > 0 {
|
||||
return hdr
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// trustedRetryAfterError annotates a terminal upstream rejection whose
|
||||
// locally recorded cooldown deadline governs the client-visible Retry-After.
|
||||
// The wrapped error keeps its own status, body, identity and retry hint; the
|
||||
@@ -507,6 +588,10 @@ func HasTrustedRetryDeadline(err error) bool {
|
||||
if SafeResponseHeaders(err).Get("Retry-After") != "" {
|
||||
return true
|
||||
}
|
||||
var poolCooldown *poolCooldownTerminalError
|
||||
if errors.As(err, &poolCooldown) && poolCooldown != nil && !poolCooldown.deadline.IsZero() {
|
||||
return true
|
||||
}
|
||||
var trusted *trustedRetryAfterError
|
||||
return errors.As(err, &trusted) && trusted != nil && !trusted.deadline.IsZero()
|
||||
}
|
||||
@@ -518,6 +603,12 @@ func trustedRetryAfterFromError(err error) *time.Duration {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
var poolCooldown *poolCooldownTerminalError
|
||||
if errors.As(err, &poolCooldown) && poolCooldown != nil {
|
||||
if remaining := poolCooldown.RetryAfter(); remaining != nil && *remaining > 0 {
|
||||
return remaining
|
||||
}
|
||||
}
|
||||
var trusted *trustedRetryAfterError
|
||||
if errors.As(err, &trusted) && trusted != nil {
|
||||
if remaining := trusted.RetryAfter(); remaining != nil && *remaining > 0 {
|
||||
|
||||
@@ -292,6 +292,10 @@ func SafeResponseHeaders(err error) http.Header {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var poolCooldown *poolCooldownTerminalError
|
||||
if errors.As(err, &poolCooldown) && poolCooldown != nil {
|
||||
return poolCooldown.SafeResponseHeaders()
|
||||
}
|
||||
var trusted *trustedRetryAfterError
|
||||
if errors.As(err, &trusted) && trusted != nil {
|
||||
return trusted.SafeResponseHeaders()
|
||||
|
||||
Reference in New Issue
Block a user