fix(auth): rotate keys after DeepSeek insufficient balance

This commit is contained in:
sususu
2026-08-10 10:13:29 +08:00
parent ecc9aa72b3
commit 45ffd115fd
3 changed files with 106 additions and 0 deletions

View File

@@ -80,6 +80,12 @@ func IsRequestFault(status int, err error) bool {
status = statusErr.StatusCode()
}
}
// HTTP 402 is a credential payment or balance failure. Some upstreams,
// including DeepSeek, label it with the generic invalid_request_error code.
// The status is authoritative so the credential can cool down and rotate.
if status == http.StatusPaymentRequired {
return false
}
if hasRequestFaultBody(err) {
return true
}

View File

@@ -191,6 +191,12 @@ func TestIsRequestFault(t *testing.T) {
},
{name: "plain not found", status: http.StatusNotFound, err: errors.New("model not found")},
{name: "unauthorized", status: http.StatusUnauthorized, err: errors.New("invalid token")},
{
name: "deepseek insufficient balance is payment failure",
status: http.StatusPaymentRequired,
err: errors.New(`{"error":{"message":"Insufficient Balance","type":"unknown_error","param":null,"code":"invalid_request_error"}}`),
want: false,
},
{name: "quota", status: http.StatusTooManyRequests, err: errors.New("quota")},
{name: "transport", status: http.StatusBadGateway, err: errors.New("unexpected EOF")},
{name: "invalid JSON body", status: http.StatusBadGateway, err: errors.New(`{"error":`)},

View File

@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"net/http"
"slices"
"sync"
"testing"
"time"
@@ -1353,6 +1354,99 @@ func TestManager_RequestScopedErrorStopsCredentialFallbackWithoutSuspendingAuth(
}
}
func TestManager_DeepSeekInsufficientBalanceRotatesCredentialAndRebindsSession(t *testing.T) {
m := NewManager(nil, nil, nil)
m.SetRetryConfig(2, 30*time.Second, 0)
affinity := NewSessionAffinitySelectorWithConfig(SessionAffinityConfig{
Fallback: &RoundRobinSelector{},
TTL: time.Hour,
})
defer affinity.Stop()
m.SetSelector(affinity)
const provider = "openai-compatibility"
const model = "deepseek-v4-pro"
executor := &authFallbackExecutor{
id: provider,
executeErrors: map[string]error{
"aa-empty-balance": &Error{
HTTPStatus: http.StatusPaymentRequired,
Message: `{"error":{"message":"Insufficient Balance","type":"unknown_error","param":null,"code":"invalid_request_error"}}`,
},
},
}
m.RegisterExecutor(executor)
depletedAuth := &Auth{ID: "aa-empty-balance", Provider: provider}
availableAuth := &Auth{ID: "bb-available-balance", Provider: provider}
reg := registry.GetGlobalRegistry()
models := []*registry.ModelInfo{{ID: model}}
reg.RegisterClient(depletedAuth.ID, provider, models)
reg.RegisterClient(availableAuth.ID, provider, models)
t.Cleanup(func() {
reg.UnregisterClient(depletedAuth.ID)
reg.UnregisterClient(availableAuth.ID)
})
if _, errRegister := m.Register(context.Background(), depletedAuth); errRegister != nil {
t.Fatalf("register depleted auth: %v", errRegister)
}
if _, errRegister := m.Register(context.Background(), availableAuth); errRegister != nil {
t.Fatalf("register available auth: %v", errRegister)
}
opts := cliproxyexecutor.Options{Metadata: map[string]any{
cliproxyexecutor.DerivedSessionIDMetadataKey: "deepseek-insufficient-balance",
}}
beforeExecute := time.Now()
resp, errExecute := m.Execute(
context.Background(),
[]string{provider},
cliproxyexecutor.Request{Model: model},
opts,
)
if errExecute != nil {
t.Fatalf("expected fallback to the next credential, got error: %v", errExecute)
}
if got := string(resp.Payload); got != availableAuth.ID {
t.Fatalf("served by %q, want %q", got, availableAuth.ID)
}
resp, errExecute = m.Execute(
context.Background(),
[]string{provider},
cliproxyexecutor.Request{Model: model},
opts,
)
if errExecute != nil {
t.Fatalf("expected rebound session to use the next credential, got error: %v", errExecute)
}
if got := string(resp.Payload); got != availableAuth.ID {
t.Fatalf("rebound session served by %q, want %q", got, availableAuth.ID)
}
wantCalls := []string{depletedAuth.ID, availableAuth.ID, availableAuth.ID}
if calls := executor.ExecuteCalls(); !slices.Equal(calls, wantCalls) {
t.Fatalf("credential calls = %v, want %v", calls, wantCalls)
}
updatedDepleted, ok := m.GetByID(depletedAuth.ID)
if !ok || updatedDepleted == nil {
t.Fatal("expected depleted auth to remain registered")
}
state := updatedDepleted.ModelStates[model]
if state == nil {
t.Fatal("expected the depleted credential to be cooled down for the model")
}
if !state.Unavailable {
t.Fatal("expected the depleted credential to be unavailable for the model")
}
if state.NextRetryAfter.Before(beforeExecute.Add(29 * time.Minute)) {
t.Fatalf("cooldown expires at %v, want approximately 30 minutes", state.NextRetryAfter)
}
}
// TestManager_UnknownUpstreamErrorRotatesAndPenalizesModelOnly pins the upstream
// 500 "status":"UNKNOWN" contract. It is an upstream internal failure, not a
// request fault, so the request must fall through to the next credential. The