diff --git a/internal/clienterror/client_error.go b/internal/clienterror/client_error.go index 3a2f532d8..78565428d 100644 --- a/internal/clienterror/client_error.go +++ b/internal/clienterror/client_error.go @@ -80,6 +80,12 @@ func IsRequestFault(status int, err error) bool { status = statusErr.StatusCode() } } + // HTTP 402 is a credential payment or balance failure. Some upstreams, + // including DeepSeek, label it with the generic invalid_request_error code. + // The status is authoritative so the credential can cool down and rotate. + if status == http.StatusPaymentRequired { + return false + } if hasRequestFaultBody(err) { return true } diff --git a/internal/clienterror/client_error_test.go b/internal/clienterror/client_error_test.go index db17df399..7086230b6 100644 --- a/internal/clienterror/client_error_test.go +++ b/internal/clienterror/client_error_test.go @@ -191,6 +191,12 @@ func TestIsRequestFault(t *testing.T) { }, {name: "plain not found", status: http.StatusNotFound, err: errors.New("model not found")}, {name: "unauthorized", status: http.StatusUnauthorized, err: errors.New("invalid token")}, + { + name: "deepseek insufficient balance is payment failure", + status: http.StatusPaymentRequired, + err: errors.New(`{"error":{"message":"Insufficient Balance","type":"unknown_error","param":null,"code":"invalid_request_error"}}`), + want: false, + }, {name: "quota", status: http.StatusTooManyRequests, err: errors.New("quota")}, {name: "transport", status: http.StatusBadGateway, err: errors.New("unexpected EOF")}, {name: "invalid JSON body", status: http.StatusBadGateway, err: errors.New(`{"error":`)}, diff --git a/sdk/cliproxy/auth/conductor_overrides_test.go b/sdk/cliproxy/auth/conductor_overrides_test.go index de35ebc9a..96996ee53 100644 --- a/sdk/cliproxy/auth/conductor_overrides_test.go +++ b/sdk/cliproxy/auth/conductor_overrides_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "slices" "sync" "testing" "time" @@ -1353,6 +1354,99 @@ func TestManager_RequestScopedErrorStopsCredentialFallbackWithoutSuspendingAuth( } } +func TestManager_DeepSeekInsufficientBalanceRotatesCredentialAndRebindsSession(t *testing.T) { + m := NewManager(nil, nil, nil) + m.SetRetryConfig(2, 30*time.Second, 0) + affinity := NewSessionAffinitySelectorWithConfig(SessionAffinityConfig{ + Fallback: &RoundRobinSelector{}, + TTL: time.Hour, + }) + defer affinity.Stop() + m.SetSelector(affinity) + + const provider = "openai-compatibility" + const model = "deepseek-v4-pro" + + executor := &authFallbackExecutor{ + id: provider, + executeErrors: map[string]error{ + "aa-empty-balance": &Error{ + HTTPStatus: http.StatusPaymentRequired, + Message: `{"error":{"message":"Insufficient Balance","type":"unknown_error","param":null,"code":"invalid_request_error"}}`, + }, + }, + } + m.RegisterExecutor(executor) + + depletedAuth := &Auth{ID: "aa-empty-balance", Provider: provider} + availableAuth := &Auth{ID: "bb-available-balance", Provider: provider} + + reg := registry.GetGlobalRegistry() + models := []*registry.ModelInfo{{ID: model}} + reg.RegisterClient(depletedAuth.ID, provider, models) + reg.RegisterClient(availableAuth.ID, provider, models) + t.Cleanup(func() { + reg.UnregisterClient(depletedAuth.ID) + reg.UnregisterClient(availableAuth.ID) + }) + + if _, errRegister := m.Register(context.Background(), depletedAuth); errRegister != nil { + t.Fatalf("register depleted auth: %v", errRegister) + } + if _, errRegister := m.Register(context.Background(), availableAuth); errRegister != nil { + t.Fatalf("register available auth: %v", errRegister) + } + + opts := cliproxyexecutor.Options{Metadata: map[string]any{ + cliproxyexecutor.DerivedSessionIDMetadataKey: "deepseek-insufficient-balance", + }} + beforeExecute := time.Now() + resp, errExecute := m.Execute( + context.Background(), + []string{provider}, + cliproxyexecutor.Request{Model: model}, + opts, + ) + if errExecute != nil { + t.Fatalf("expected fallback to the next credential, got error: %v", errExecute) + } + if got := string(resp.Payload); got != availableAuth.ID { + t.Fatalf("served by %q, want %q", got, availableAuth.ID) + } + + resp, errExecute = m.Execute( + context.Background(), + []string{provider}, + cliproxyexecutor.Request{Model: model}, + opts, + ) + if errExecute != nil { + t.Fatalf("expected rebound session to use the next credential, got error: %v", errExecute) + } + if got := string(resp.Payload); got != availableAuth.ID { + t.Fatalf("rebound session served by %q, want %q", got, availableAuth.ID) + } + wantCalls := []string{depletedAuth.ID, availableAuth.ID, availableAuth.ID} + if calls := executor.ExecuteCalls(); !slices.Equal(calls, wantCalls) { + t.Fatalf("credential calls = %v, want %v", calls, wantCalls) + } + + updatedDepleted, ok := m.GetByID(depletedAuth.ID) + if !ok || updatedDepleted == nil { + t.Fatal("expected depleted auth to remain registered") + } + state := updatedDepleted.ModelStates[model] + if state == nil { + t.Fatal("expected the depleted credential to be cooled down for the model") + } + if !state.Unavailable { + t.Fatal("expected the depleted credential to be unavailable for the model") + } + if state.NextRetryAfter.Before(beforeExecute.Add(29 * time.Minute)) { + t.Fatalf("cooldown expires at %v, want approximately 30 minutes", state.NextRetryAfter) + } +} + // TestManager_UnknownUpstreamErrorRotatesAndPenalizesModelOnly pins the upstream // 500 "status":"UNKNOWN" contract. It is an upstream internal failure, not a // request fault, so the request must fall through to the next credential. The